Upload folder using huggingface_hub

.bandit.yml

@@ -0,0 +1,11 @@
+skips:
+- B101
+- B311
+- B113  # `Requests call without timeout` these requests are done in the benchmark and examples scripts only
+- B403  # We are using pickle for tests only
+- B404  # Using subprocess library
+- B602  # subprocess call with shell=True identified
+- B110  # Try, Except, Pass detected.
+- B104  # Possible binding to all interfaces.
+- B301  # Pickle and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue.
+- B108  # Probable insecure usage of temp file/directory.

.dockerignore

@@ -1,12 +1,110 @@
-.git
-.gitignore
-__pycache__
-*.pyc
-*.pyo
-venv
-.venv
+# Github
+.github/
+
+# docs
+docs/
+images/
+.cache/
+.claude/
+
+# cached files
+__pycache__/
+*.py[cod]
+.cache
+.DS_Store
+*~
+.*.sw[po]
+.build
+.ve
 .env
+.pytest
+.benchmarks
+.bootstrap
+.appveyor.token
+*.bak
+*.db
+*.db-*
+
+# installation package
+*.egg-info/
+dist/
+build/
+
+# environments
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# C extensions
+*.so
+
+# pycharm
+.idea/
+
+# vscode
+*.code-workspace
+
+# Packages
+*.egg
 *.egg-info
-.DS_Store
-chat_history.json
-client_secret.json
+dist
+build
+eggs
+.eggs
+parts
+bin
+var
+sdist
+wheelhouse
+develop-eggs
+.installed.cfg
+lib
+lib64
+venv*/
+.venv*/
+pyvenv*/
+pip-wheel-metadata/
+poetry.lock
+
+# Installer logs
+pip-log.txt
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+mypy.ini
+
+# test caches
+.tox/
+.pytest_cache/
+.coverage
+htmlcov
+report.xml
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Buildout
+.mr.developer.cfg
+
+# IDE project files
+.project
+.pydevproject
+.idea
+*.iml
+*.komodoproject
+
+# Complexity
+output/*.html
+output/*/index.html
+
+# Sphinx
+docs/_build
+public/
+web/

.gitattributes

@@ -3,3 +3,8 @@ Scrapling/docs/assets/favicon.ico filter=lfs diff=lfs merge=lfs -text
 Scrapling/docs/assets/main_cover.png filter=lfs diff=lfs merge=lfs -text
 Scrapling/docs/assets/scrapling_shell_curl.png filter=lfs diff=lfs merge=lfs -text
 Scrapling/docs/assets/spider_architecture.png filter=lfs diff=lfs merge=lfs -text
+docs/assets/cover_dark.png filter=lfs diff=lfs merge=lfs -text
+docs/assets/favicon.ico filter=lfs diff=lfs merge=lfs -text
+docs/assets/main_cover.png filter=lfs diff=lfs merge=lfs -text
+docs/assets/scrapling_shell_curl.png filter=lfs diff=lfs merge=lfs -text
+docs/assets/spider_architecture.png filter=lfs diff=lfs merge=lfs -text

.github/FUNDING.yml

@@ -1,3 +1,3 @@
-# These are supported funding model platforms
-
-github: itsOwen
+github: D4Vinci
+buy_me_a_coffee: d4vinci
+ko_fi: d4vinci

.github/ISSUE_TEMPLATE/01-bug_report.yml

@@ -0,0 +1,82 @@
+name: Bug report
+description: Create a bug report to help us address errors in the repository
+labels: [bug]
+body:
+  - type: checkboxes
+    attributes:
+      label: Have you searched if there an existing issue for this?
+      description: Please search [existing issues](https://github.com/D4Vinci/Scrapling/labels/bug).
+      options:
+        - label: I have searched the existing issues
+          required: true
+
+  - type: input
+    attributes:
+      label: "Python version (python --version)"
+      placeholder: "Python 3.8"
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: "Scrapling version (scrapling.__version__)"
+      placeholder: "0.1"
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: "Dependencies version (pip3 freeze)"
+      description: >
+        This is the output of the command `pip3 freeze --all`. Note that the
+        actual output might be different as compared to the placeholder text.
+      placeholder: |
+        cssselect==1.2.0
+        lxml==5.3.0
+        orjson==3.10.7
+        ...
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: "What's your operating system?"
+      placeholder: "Windows 10"
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: 'Are you using a separate virtual environment?'
+      description: "Please pay attention to this question"
+      options:
+        - 'No'
+        - 'Yes'
+      default: 0
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: "Expected behavior"
+      description: "Describe the behavior you expect. May include images or videos."
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: "Actual behavior"
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Steps To Reproduce
+      description: Steps to reproduce the behavior.
+      placeholder: |
+        1. In this environment...
+        2. With this config...
+        3. Run '...'
+        4. See error...
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/02-feature_request.yml

@@ -0,0 +1,19 @@
+name: Feature request
+description: Suggest features, propose improvements, discuss new ideas.
+labels: [enhancement]
+body:
+  - type: checkboxes
+    attributes:
+      label: Have you searched if there an existing feature request for this?
+      description: Please search [existing requests](https://github.com/D4Vinci/Scrapling/labels/enhancement).
+      options:
+        - label: I have searched the existing requests
+          required: true
+
+  - type: textarea
+    attributes:
+      label: "Feature description"
+      description: >
+        This could include new topics or improving any existing features/implementations.
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/03-other.yml

@@ -0,0 +1,19 @@
+name: Other
+description: Use this for any other issues. PLEASE provide as much information as possible.
+labels: ["awaiting triage"]
+body:
+  - type: textarea
+    id: issuedescription
+    attributes:
+      label: What would you like to share?
+      description: Provide a clear and concise explanation of your issue.
+    validations:
+      required: true
+
+  - type: textarea
+    id: extrainfo
+    attributes:
+      label: Additional information
+      description: Is there anything else we should know about this issue?
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/04-docs_issue.yml

@@ -0,0 +1,40 @@
+name: Documentation issue
+description: Report incorrect, unclear, or missing documentation.
+labels: [documentation]
+body:
+  - type: checkboxes
+    attributes:
+      label: Have you searched if there an existing issue for this?
+      description: Please search [existing issues](https://github.com/D4Vinci/Scrapling/labels/documentation).
+      options:
+        - label: I have searched the existing issues
+          required: true
+
+  - type: input
+    attributes:
+      label: "Page URL"
+      description: "Link to the documentation page with the issue."
+      placeholder: "https://scrapling.readthedocs.io/en/latest/..."
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: "Type of issue"
+      options:
+        - Incorrect information
+        - Unclear or confusing
+        - Missing information
+        - Typo or formatting
+        - Broken link
+        - Other
+      default: 0
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: "Description"
+      description: "Describe what's wrong and what you expected to find."
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,10 @@
+blank_issues_enabled: false
+contact_links:
+- name: Discussions
+  url: https://github.com/D4Vinci/Scrapling/discussions
+  about: >
+    The "Discussions" forum is where you want to start. 💖
+- name: Ask on our discord server
+  url: https://discord.gg/EMgGbDceNQ
+  about: >
+    Our community chat forum.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,51 @@
+<!--
+  You are amazing! Thanks for contributing to Scrapling!
+  Please, DO NOT DELETE ANY TEXT from this template! (unless instructed).
+-->
+
+## Proposed change
+<!--
+  Describe the big picture of your changes here to communicate to the maintainers why we should accept this pull request.
+  If it fixes a bug or resolves a feature request, be sure to link to that issue in the additional information section.
+-->
+
+
+### Type of change:
+<!--
+  What type of change does your PR introduce to Scrapling?
+  NOTE: Please, check at least 1 box!
+  If your PR requires multiple boxes to be checked, you'll most likely need to
+  split it into multiple PRs. This makes things easier and faster to code review.
+-->
+
+
+
+- [ ] Dependency upgrade
+- [ ] Bugfix (non-breaking change which fixes an issue)
+- [ ] New integration (thank you!)
+- [ ] New feature (which adds functionality to an existing integration)
+- [ ] Deprecation (breaking change to happen in the future)
+- [ ] Breaking change (fix/feature causing existing functionality to break)
+- [ ] Code quality improvements to existing code or addition of tests
+- [ ] Add or change doctests? -- Note: Please avoid changing both code and tests in a single pull request.
+- [ ] Documentation change?
+
+### Additional information
+<!--
+  Details are important and help maintainers processing your PR.
+  Please be sure to fill out additional details, if applicable.
+-->
+
+- This PR fixes or closes an issue: fixes #
+- This PR is related to an issue: #
+- Link to documentation pull request: **
+
+### Checklist:
+* [ ] I have read [CONTRIBUTING.md](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md).
+* [ ] This pull request is all my own work -- I have not plagiarized.
+* [ ] I know that pull requests will not be merged if they fail the automated tests.
+* [ ] All new Python files are placed inside an existing directory.
+* [ ] All filenames are in all lowercase characters with no spaces or dashes.
+* [ ] All functions and variable names follow Python naming conventions.
+* [ ] All function parameters and return values are annotated with Python [type hints](https://docs.python.org/3/library/typing.html).
+* [ ] All functions have doc-strings.

.github/workflows/code-quality.yml

@@ -0,0 +1,184 @@
+name: Code Quality
+
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths-ignore:
+      - '*.md'
+      - '**/*.md'
+      - 'docs/**'
+      - 'images/**'
+      - '.github/**'
+      - '!.github/workflows/code-quality.yml'  # Always run when this workflow changes
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths-ignore:
+      - '*.md'
+      - '**/*.md'
+      - 'docs/**'
+      - 'images/**'
+  workflow_dispatch:  # Allow manual triggering
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  code-quality:
+    name: Code Quality Checks
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write  # For PR annotations
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0  # Full history for better analysis
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install bandit[toml] ruff vermin mypy pyright
+          pip install -e ".[all]"
+          pip install lxml-stubs
+
+      - name: Run Bandit (Security Linter)
+        id: bandit
+        continue-on-error: true
+        run: |
+          echo "::group::Bandit - Security Linter"
+          bandit -r -c .bandit.yml scrapling/ -f json -o bandit-report.json
+          bandit -r -c .bandit.yml scrapling/
+          echo "::endgroup::"
+
+      - name: Run Ruff Linter
+        id: ruff-lint
+        continue-on-error: true
+        run: |
+          echo "::group::Ruff - Linter"
+          ruff check scrapling/ --output-format=github
+          echo "::endgroup::"
+
+      - name: Run Ruff Formatter Check
+        id: ruff-format
+        continue-on-error: true
+        run: |
+          echo "::group::Ruff - Formatter Check"
+          ruff format --check scrapling/ --diff
+          echo "::endgroup::"
+
+      - name: Run Vermin (Python Version Compatibility)
+        id: vermin
+        continue-on-error: true
+        run: |
+          echo "::group::Vermin - Python 3.10+ Compatibility Check"
+          vermin -t=3.10- --violations --eval-annotations --no-tips scrapling/
+          echo "::endgroup::"
+
+      - name: Run Mypy (Static Type Checker)
+        id: mypy
+        continue-on-error: true
+        run: |
+          echo "::group::Mypy - Static Type Checker"
+          mypy scrapling/
+          echo "::endgroup::"
+
+      - name: Run Pyright (Static Type Checker)
+        id: pyright
+        continue-on-error: true
+        run: |
+          echo "::group::Pyright - Static Type Checker"
+          pyright scrapling/
+          echo "::endgroup::"
+
+      - name: Check results and create summary
+        if: always()
+        run: |
+          echo "# Code Quality Check Results" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          # Initialize status
+          all_passed=true
+
+          # Check Bandit
+          if [ "${{ steps.bandit.outcome }}" == "success" ]; then
+            echo "✅ **Bandit (Security)**: Passed" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "❌ **Bandit (Security)**: Failed" >> $GITHUB_STEP_SUMMARY
+            all_passed=false
+          fi
+
+          # Check Ruff Linter
+          if [ "${{ steps.ruff-lint.outcome }}" == "success" ]; then
+            echo "✅ **Ruff Linter**: Passed" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "❌ **Ruff Linter**: Failed" >> $GITHUB_STEP_SUMMARY
+            all_passed=false
+          fi
+
+          # Check Ruff Formatter
+          if [ "${{ steps.ruff-format.outcome }}" == "success" ]; then
+            echo "✅ **Ruff Formatter**: Passed" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "❌ **Ruff Formatter**: Failed" >> $GITHUB_STEP_SUMMARY
+            all_passed=false
+          fi
+
+          # Check Vermin
+          if [ "${{ steps.vermin.outcome }}" == "success" ]; then
+            echo "✅ **Vermin (Python 3.10+)**: Passed" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "❌ **Vermin (Python 3.10+)**: Failed" >> $GITHUB_STEP_SUMMARY
+            all_passed=false
+          fi
+
+          # Check Mypy
+          if [ "${{ steps.mypy.outcome }}" == "success" ]; then
+            echo "✅ **Mypy (Type Checker)**: Passed" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "❌ **Mypy (Type Checker)**: Failed" >> $GITHUB_STEP_SUMMARY
+            all_passed=false
+          fi
+
+          # Check Pyright
+          if [ "${{ steps.pyright.outcome }}" == "success" ]; then
+            echo "✅ **Pyright (Type Checker)**: Passed" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "❌ **Pyright (Type Checker)**: Failed" >> $GITHUB_STEP_SUMMARY
+            all_passed=false
+          fi
+
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          if [ "$all_passed" == "true" ]; then
+            echo "### 🎉 All checks passed!" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "Your code meets all quality standards." >> $GITHUB_STEP_SUMMARY
+          else
+            echo "### ⚠️ Some checks failed" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "Please review the errors above and fix them." >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "**Tip**: Run \`pre-commit run --all-files\` locally to catch these issues before pushing." >> $GITHUB_STEP_SUMMARY
+            exit 1
+          fi
+
+      - name: Upload Bandit report
+        if: always() && steps.bandit.outcome != 'skipped'
+        uses: actions/upload-artifact@v6
+        with:
+          name: bandit-security-report
+          path: bandit-report.json
+          retention-days: 30

.github/workflows/docker-build.yml

@@ -0,0 +1,86 @@
+name: Build and Push Docker Image
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - main
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Docker image tag'
+        required: true
+        default: 'latest'
+
+env:
+  DOCKERHUB_IMAGE: pyd4vinci/scrapling
+  GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/scrapling
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v6
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      with:
+        platforms: linux/amd64,linux/arm64
+
+    - name: Log in to Docker Hub
+      uses: docker/login-action@v3
+      with:
+        registry: docker.io
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+
+    - name: Log in to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.CONTAINER_TOKEN }}
+
+    - name: Extract metadata
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: |
+          ${{ env.DOCKERHUB_IMAGE }}
+          ${{ env.GHCR_IMAGE }}
+        tags: |
+          type=ref,event=branch
+          type=ref,event=pr
+          type=semver,pattern={{version}}
+          type=semver,pattern={{major}}.{{minor}}
+          type=semver,pattern={{major}}
+          type=raw,value=latest,enable={{is_default_branch}}
+        labels: |
+          org.opencontainers.image.title=Scrapling
+          org.opencontainers.image.description=An undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!
+          org.opencontainers.image.vendor=D4Vinci
+          org.opencontainers.image.licenses=BSD
+          org.opencontainers.image.url=https://scrapling.readthedocs.io/en/latest/
+          org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+          org.opencontainers.image.documentation=https://scrapling.readthedocs.io/en/latest/
+
+    - name: Build and push Docker image
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        platforms: linux/amd64,linux/arm64
+        push: true
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        build-args: |
+          BUILDKIT_INLINE_CACHE=1
+
+    - name: Image digest
+      run: echo ${{ steps.build.outputs.digest }}

.github/workflows/release-and-publish.yml

@@ -0,0 +1,74 @@
+name: Create Release and Publish to PyPI
+# Creates a GitHub release when a PR is merged to main (using PR title as version and body as release notes), then publishes to PyPI.
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - main
+
+jobs:
+  create-release-and-publish:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    environment:
+      name: PyPI
+      url: https://pypi.org/p/scrapling
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Get PR title
+        id: pr_title
+        run: echo "title=${{ github.event.pull_request.title }}" >> $GITHUB_OUTPUT
+
+      - name: Save PR body to file
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const fs = require('fs');
+            fs.writeFileSync('pr_body.md', context.payload.pull_request.body || '');
+
+      - name: Extract version
+        id: extract_version
+        run: |
+          PR_TITLE="${{ steps.pr_title.outputs.title }}"
+          if [[ $PR_TITLE =~ ^v ]]; then
+            echo "version=$PR_TITLE" >> $GITHUB_OUTPUT
+            echo "Valid version format found in PR title: $PR_TITLE"
+          else
+            echo "Error: PR title '$PR_TITLE' must start with 'v' (e.g., 'v1.0.0') to create a release."
+            exit 1
+          fi
+
+      - name: Create Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ steps.extract_version.outputs.version }}
+          name: Release ${{ steps.extract_version.outputs.version }}
+          body_path: pr_body.md
+          draft: false
+          prerelease: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.12
+
+      - name: Upgrade pip
+        run: python3 -m pip install --upgrade pip
+
+      - name: Install build
+        run: python3 -m pip install --upgrade build twine setuptools
+
+      - name: Build a binary wheel and a source tarball
+        run: python3 -m build --sdist --wheel --outdir dist/
+
+      - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/tests.yml

@@ -0,0 +1,109 @@
+name: Tests
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths-ignore:
+      - '*.md'
+      - '**/*.md'
+      - 'docs/*'
+      - 'images/*'
+      - '.github/*'
+      - '*.yml'
+      - '*.yaml'
+      - 'ruff.toml'
+
+concurrency:
+  group: ${{github.workflow}}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    timeout-minutes: 60
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - python-version: "3.10"
+          os: macos-latest
+          env:
+            TOXENV: py310
+        - python-version: "3.11"
+          os: macos-latest
+          env:
+            TOXENV: py311
+        - python-version: "3.12"
+          os: macos-latest
+          env:
+            TOXENV: py312
+        - python-version: "3.13"
+          os: macos-latest
+          env:
+            TOXENV: py313
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          pyproject.toml
+          tox.ini
+
+    - name: Install all browsers dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        python3 -m pip install playwright==1.56.0 patchright==1.56.0
+
+    - name: Get Playwright version
+      id: playwright-version
+      run: |
+        PLAYWRIGHT_VERSION=$(python3 -c "import importlib.metadata; print(importlib.metadata.version('playwright'))")
+        echo "version=$PLAYWRIGHT_VERSION" >> $GITHUB_OUTPUT
+        echo "Playwright version: $PLAYWRIGHT_VERSION"
+
+    - name: Retrieve Playwright browsers from cache if any
+      id: playwright-cache
+      uses: actions/cache@v5
+      with:
+        path: |
+          ~/.cache/ms-playwright
+          ~/Library/Caches/ms-playwright
+          ~/.ms-playwright
+        key: ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}-v1
+        restore-keys: |
+          ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}-
+          ${{ runner.os }}-playwright-
+
+    - name: Install Playwright browsers
+      run: |
+        echo "Cache hit: ${{ steps.playwright-cache.outputs.cache-hit }}"
+        if [ "${{ steps.playwright-cache.outputs.cache-hit }}" != "true" ]; then
+          python3 -m playwright install chromium
+        else
+          echo "Skipping install - using cached Playwright browsers"
+        fi
+        python3 -m playwright install-deps chromium
+
+    # Cache tox environments
+    - name: Cache tox environments
+      uses: actions/cache@v5
+      with:
+        path: .tox
+        # Include python version and os in the cache key
+        key: tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('/Users/runner/work/Scrapling/pyproject.toml') }}
+        restore-keys: |
+          tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-
+          tox-v1-${{ runner.os }}-
+
+    - name: Install tox
+      run: pip install -U tox
+
+    - name: Run tests
+      env: ${{ matrix.env }}
+      run: tox

.gitignore

@@ -1,75 +1,110 @@
-# Python cache files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# Virtual environment
-venv/
-
-# Streamlit cache
-.streamlit/
-
-# PyCharm files
-.idea/
-
-# VS Code files
-.vscode/
+# local files
+site/*
+local_tests/*
+.mcpregistry_*
 
-# Jupyter Notebook
-.ipynb_checkpoints
+# AI related files
+.claude/*
+CLAUDE.md
 
-# Environment variables
-.env
-
-# Operating system files
+# cached files
+__pycache__/
+*.py[cod]
+.cache
 .DS_Store
-Thumbs.db
-
-# Log files
-*.log
-
-# Database files
+*~
+.*.sw[po]
+.build
+.ve
+.env
+.pytest
+.benchmarks
+.bootstrap
+.appveyor.token
+*.bak
 *.db
-*.sqlite3
+*.db-*
 
-# Compiled Python files
-*.pyc
-
-# Package directories
+# installation package
+*.egg-info/
 dist/
 build/
-*.egg-info/
 
-# Backup files
-*~
-*.bak
+# environments
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
 
-# Coverage reports
-htmlcov/
-.coverage
-.coverage.*
-coverage.xml
+# C extensions
+*.so
 
-# Pytest cache
-.pytest_cache/
+# pycharm
+.idea/
 
-# mypy cache
+# vscode
+*.code-workspace
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+.eggs
+parts
+bin
+var
+sdist
+wheelhouse
+develop-eggs
+.installed.cfg
+lib
+lib64
+venv*/
+.venv*/
+pyvenv*/
+pip-wheel-metadata/
+poetry.lock
+
+# Installer logs
+pip-log.txt
+
+# mypy
 .mypy_cache/
+.dmypy.json
+dmypy.json
+mypy.ini
 
-# Scrapy stuff:
-.scrapy
+# test caches
+.tox/
+.pytest_cache/
+.coverage
+htmlcov
+report.xml
+nosetests.xml
+coverage.xml
 
-# Sphinx documentation
-docs/_build/
+# Translations
+*.mo
 
-# PyBuilder
-target/
+# Buildout
+.mr.developer.cfg
 
-# Google Sheets authentication token
-token.json
+# IDE project files
+.project
+.pydevproject
+.idea
+*.iml
+*.komodoproject
 
-# Chat history
-chat_history.json
+# Complexity
+output/*.html
+output/*/index.html
 
-# Google OAuth client secret
-client_secret.json
+# Sphinx
+docs/_build
+public/
+web/

.hfignore

@@ -1,9 +1,23 @@
-.git/
-.github/
-venv/
-__pycache__/
+.git
+.github
+.venv
+__pycache__
 *.pyc
+*.pyo
+*.pyd
+.DS_Store
+tests/
+docs/
+images/
+.coverage
+htmlcov/
+pytest_cache/
+.mypy_cache/
+.tox/
+.pytest_cache/
+.ruff_cache/
+.uv/
+dist/
+build/
+*.egg-info/
 .env
-chat_history.json
-test_patchright.py
-client_secret.json

.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+repos:
+- repo: https://github.com/PyCQA/bandit
+  rev: 1.9.0
+  hooks:
+  - id: bandit
+    args: [-r, -c, .bandit.yml]
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.14.5
+  hooks:
+    # Run the linter.
+    - id: ruff
+      args: [ --fix ]
+    # Run the formatter.
+    - id: ruff-format
+- repo: https://github.com/netromdk/vermin
+  rev: v1.7.0
+  hooks:
+  - id: vermin
+    args: ['-t=3.10-', '--violations', '--eval-annotations', '--no-tips']

.readthedocs.yaml

@@ -0,0 +1,21 @@
+# See https://docs.readthedocs.com/platform/stable/intro/zensical.html for details
+# Example: https://github.com/readthedocs/test-builds/tree/zensical
+
+version: 2
+
+build:
+  os: ubuntu-24.04
+  apt_packages:
+    - pngquant
+  tools:
+    python: "3.13"
+  jobs:
+    install:
+      - pip install -r docs/requirements.txt
+      - pip install ".[all]"
+    build:
+      html:
+        - zensical build
+    post_build:
+      - mkdir -p $READTHEDOCS_OUTPUT/html/
+      - cp --recursive site/* $READTHEDOCS_OUTPUT/html/

CODE_OF_CONDUCT.md

@@ -60,7 +60,7 @@ representative at an online or offline event.
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
 reported to the community leaders responsible for enforcement at
-owensingh72@gmail.com.
+karim.shoair@pm.me.
 All complaints will be reviewed and investigated promptly and fairly.
 
 All community leaders are obligated to respect the privacy and security of the

CONTRIBUTING.md

@@ -1,167 +1,106 @@
-# Contributing to CyberScraper 2077
+# Contributing to Scrapling
 
-> "In 2077, what makes someone a contributor? Pushing code." - Johnny Silverhand
+Thank you for your interest in contributing to Scrapling! 
 
-Thanks for considering contributing to CyberScraper 2077! This document outlines the process and guidelines for contributing to make the experience smooth for everyone involved.
+Everybody is invited and welcome to contribute to Scrapling. 
 
-## 🤝 Code of Conduct
+Minor changes are more likely to be included promptly. Adding unit tests for new features or test cases for bugs you've fixed helps us ensure that the Pull Request (PR) is acceptable.
 
-By participating in this project, you agree to abide by our Code of Conduct. Please read it before contributing.
+There are many ways to contribute to Scrapling. Here are some of them:
 
-## 🚀 How to Contribute
+- Report bugs and request features using the [GitHub issues](https://github.com/D4Vinci/Scrapling/issues). Please follow the issue template to help us resolve your issue quickly.
+- Blog about Scrapling. Tell the world how you’re using Scrapling. This will help newcomers with more examples and increase the Scrapling project's visibility.
+- Join the [Discord community](https://discord.gg/EMgGbDceNQ) and share your ideas on how to improve Scrapling. We’re always open to suggestions.
+- If you are not a developer, perhaps you would like to help with translating the [documentation](https://github.com/D4Vinci/Scrapling/tree/docs)?
 
-### Setting Up Development Environment
 
-1. Fork the repository
-2. Clone your fork:
-   ```bash
-   git clone https://github.com/your-username/CyberScraper-2077.git
-   cd CyberScraper-2077
-   ```
-3. Create a virtual environment:
-   ```bash
-   python -m venv venv
-   source venv/bin/activate  # On Windows: venv\Scripts\activate
-   ```
-4. Install dependencies:
-   ```bash
-   pip install -r requirements.txt
-   playwright install
-   ```
+## Finding work
 
-### Making Changes
+If you have decided to make a contribution to Scrapling, but you do not know what to contribute, here are some ways to find pending work:
 
-1. Create a new branch:
-   ```bash
-   git checkout -b feature/your-feature-name
-   ```
-2. Make your changes
-3. Test your changes thoroughly
-4. Commit your changes:
-   ```bash
-   git commit -m "feat: add new feature"
+- Check out the [contribution](https://github.com/D4Vinci/Scrapling/contribute) GitHub page, which lists open issues tagged as `good first issue`. These issues provide a good starting point.
+- There are also the [help wanted](https://github.com/D4Vinci/Scrapling/issues?q=is%3Aissue%20label%3A%22help%20wanted%22%20state%3Aopen) issues, but know that some may require familiarity with the Scrapling code base first. You can also target any other issue, provided it is not tagged as `invalid`, `wontfix`, or similar tags.
+- If you enjoy writing automated tests, you can work on increasing our test coverage. Currently, the test coverage is around 90–92%.
+- Join the [Discord community](https://discord.gg/EMgGbDceNQ) and ask questions in the `#help` channel.
+
+## Coding style
+Please follow these coding conventions as we do when writing code for Scrapling:
+- We use [pre-commit](https://pre-commit.com/) to automatically address simple code issues before every commit, so please install it and run `pre-commit install` to set it up. This will install hooks to run [ruff](https://docs.astral.sh/ruff/), [bandit](https://github.com/PyCQA/bandit), and [vermin](https://github.com/netromdk/vermin) on every commit. We are currently using a workflow to automatically run these tools on every PR, so if your code doesn't pass these checks, the PR will be rejected.
+- We use type hints for better code clarity and [pyright](https://github.com/microsoft/pyright) for static type checking, which depends on the type hints, of course.
+- We use the conventional commit messages format as [here](https://gist.github.com/qoomon/5dfcdf8eec66a051ecd85625518cfd13#types), so for example, we use the following prefixes for commit messages:
+   
+   | Prefix      | When to use it           |
+   |-------------|--------------------------|
+   | `feat:`     | New feature added        |
+   | `fix:`      | Bug fix                  |
+   | `docs:`     | Documentation change/add |
+   | `test:`     | Tests                    |
+   | `refactor:` | Code refactoring         |
+   | `chore:`    | Maintenance tasks        |
+    
+    Then include the details of the change in the commit message body/description.
+
+   Example:
    ```
-5. Push to your fork:
-   ```bash
-   git push origin feature/your-feature-name
+   feat: add `adaptive` for similar elements
+   
+   - Added find_similar() method
+   - Implemented pattern matching
+   - Added tests and documentation
    ```
-6. Create a Pull Request
-
-## 📝 Commit Message Guidelines
 
-We follow [Conventional Commits](https://www.conventionalcommits.org/). Your commit messages should be structured as follows:
+> Please don’t put your name in the code you contribute; git provides enough metadata to identify the author of the code.
 
+## Development
+Setting the scrapling logging level to `debug` makes it easier to know what's happening in the background.
+```python
+import logging
+logging.getLogger("scrapling").setLevel(logging.DEBUG)
 ```
-<type>(<scope>): <description>
-
-[optional body]
-
-[optional footer]
+Bonus: You can install the beta of the upcoming update from the dev branch as follows
+```commandline
+pip3 install git+https://github.com/D4Vinci/Scrapling.git@dev
 ```
 
-Types:
-- `feat`: New feature
-- `fix`: Bug fix
-- `docs`: Documentation changes
-- `style`: Code style changes (formatting, missing semi-colons, etc)
-- `refactor`: Code refactoring
-- `test`: Adding missing tests
-- `chore`: Changes to build process or auxiliary tools
-
-Example:
-```
-feat(scraper): add support for dynamic loading websites
+## Building Documentation
+Documentation is built using [MkDocs](https://www.mkdocs.org/). You can build it locally using the following commands:
+```bash
+pip install mkdocs-material
+mkdocs serve  # Local preview
+mkdocs build  # Build the static site
 ```
 
-## 🧪 Testing Guidelines
-
-- Write tests for new features
-- Ensure all tests pass before submitting PR
-- Follow existing test patterns
-- Include both unit and integration tests when applicable
-
-## 📚 Documentation Guidelines
-
-- Update README.md if adding new features
-- Add docstrings to new functions/classes
-- Include code examples when appropriate
-- Keep documentation clear and concise
-
-## 🏗️ Project Structure
+## Tests
+Scrapling includes a comprehensive test suite that can be executed with pytest. However, first, you need to install all libraries and `pytest-plugins` listed in `tests/requirements.txt`. Then, running the tests will result in an output like this:
+   ```bash
+   $ pytest tests -n auto
+   =============================== test session starts ===============================
+   platform darwin -- Python 3.13.8, pytest-8.4.2, pluggy-1.6.0 -- /Users/<redacted>/.venv/bin/python3.13
+   cachedir: .pytest_cache
+   rootdir: /Users/<redacted>/scrapling
+   configfile: pytest.ini
+   plugins: asyncio-1.2.0, anyio-4.11.0, xdist-3.8.0, httpbin-2.1.0, cov-7.0.0
+   asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function
+   10 workers [271 items]    
+   scheduling tests via LoadScheduling 
+   
+   ...<shortened>...
+   
+   =============================== 271 passed in 52.68s ==============================
+   ```
+Hence, we used `-n auto` in the command above to run tests in threads to increase speed.
 
+Bonus: You can also see the test coverage with the `pytest` plugin below
+```bash
+pytest --cov=scrapling tests/
 ```
-CyberScraper-2077/
-├── app/
-│   ├── scrapers/
-│   ├── utils/
-│   └── ui_components/
-├── src/
-│   └── models/
-├── tests/
-└── docs/
-```
-
-- Place new scraper implementations in `app/scrapers/`
-- Add utility functions in `app/utils/`
-- UI components go in `app/ui_components/`
-- Model-related code goes in `src/models/`
-
-## 🎯 Feature Requests
-
-- Use GitHub Issues to propose new features
-- Tag feature requests with `enhancement`
-- Provide clear use cases
-- Discuss implementation approach
-
-## 🐛 Bug Reports
-
-When reporting bugs, include:
-- Detailed description of the issue
-- Steps to reproduce
-- Expected vs actual behavior
-- Environment details (OS, Python version, etc.)
-- Screenshots if applicable
-
-## 🔍 Pull Request Process
-
-1. Update documentation
-2. Add/update tests
-3. Ensure CI/CD pipeline passes
-4. Get at least one code review
-5. Squash commits if requested
-6. Ensure branch is up to date with main
-
-## ⚙️ Development Best Practices
-
-1. Follow PEP 8 style guide
-2. Use type hints
-3. Keep functions/methods focused and small
-4. Comment complex logic
-5. Use meaningful variable/function names
-6. Handle errors appropriately
-7. Log important operations
-
-## 🚫 What to Avoid
-
-- Breaking existing functionality
-- Introducing unnecessary dependencies
-- Making large, unfocused PRs
-- Ignoring code review feedback
-- Modifying core functionality without discussion
-
-## 🏆 Recognition
-
-Contributors will be added to our README.md and CONTRIBUTORS.md files. We value and appreciate all contributions!
-
-## 📞 Getting Help
-
-- Create an issue for questions
-- Join our Discord community
-- Check existing documentation
-- Look through closed issues
-
-## 📜 License
 
-By contributing, you agree that your contributions will be licensed under the project's MIT License.
+## Making a Pull Request
+To ensure that your PR gets accepted, please make sure that your PR is based on the latest changes from the dev branch and that it satisfies the following requirements:
 
-Remember: In Night City - and in open source - style is everything, choom. Let's keep the code clean and the commits conventional.
+- The PR should be made against the [**dev**](https://github.com/D4Vinci/Scrapling/tree/dev) branch of Scrapling. Any PR made against the main branch will be rejected.
+- The code should be passing all available tests. We use tox with GitHub's CI to run the current tests on all supported Python versions for every code-related commit.
+- The code should be passing all code quality checks we mentioned above. We are using GitHub's CI to enforce the code style checks performed by pre-commit. If you were using the pre-commit hooks we discussed above, you should not see any issues when committing your changes.
+- Make your changes, keep the code clean with an explanation of any part that might be vague, and remember to create a separate virtual environment for this project.
+- If you are adding a new feature, please add tests for it.
+- If you are fixing a bug, please add code with the PR that reproduces the bug.

Dockerfile

@@ -1,102 +1,48 @@
-# Use Python 3.12 for better performance and compatibility
-FROM python:3.12-slim-bookworm
+FROM python:3.12-slim-trixie
 
-# Set environment variables
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PORT=7860 \
-    UV_SYSTEM_PYTHON=1 \
-    HOME=/home/user \
-    STREAMLIT_BROWSER_GATHER_USAGE_STATS=false \
-    STREAMLIT_SERVER_HEADLESS=true \
-    STREAMLIT_SERVER_PORT=8501 \
-    STREAMLIT_SERVER_ADDRESS=0.0.0.0
-
-# Install system dependencies
-RUN apt-get update && apt-get install -y \
-    wget \
-    gnupg \
-    git \
-    tor \
-    tor-geoipdb \
-    netcat-traditional \
-    curl \
-    build-essential \
-    python3-dev \
-    libffi-dev \
-    procps \
-    nginx \
-    # Browser dependencies for Playwright/Patchright
-    libglib2.0-0 \
-    libnspr4 \
-    libnss3 \
-    libdbus-1-3 \
-    libatk1.0-0 \
-    libatk-bridge2.0-0 \
-    libcups2 \
-    libxkbcommon0 \
-    libatspi2.0-0 \
-    libxcomposite1 \
-    libxdamage1 \
-    libxfixes3 \
-    libxrandr2 \
-    libgbm1 \
-    libcairo2 \
-    libpango-1.0-0 \
-    libasound2 \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-# Install uv
+LABEL io.modelcontextprotocol.server.name="io.github.D4Vinci/Scrapling"
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
-# Set up working directory
-WORKDIR /app
-
-# Copy requirements and install as root
-COPY requirements.txt .
-RUN uv pip install --system -r requirements.txt
-RUN uv pip install --system fastapi uvicorn
-
-# Install patchright browser (Chromium)
-RUN patchright install chromium
-
-# Create a non-root user
-RUN useradd -m -u 1000 user
+# Set environment variables
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
 
-# Configure Tor
-RUN echo "SocksPort 9050" >> /etc/tor/torrc && \
-    echo "ControlPort 9051" >> /etc/tor/torrc && \
-    echo "CookieAuthentication 1" >> /etc/tor/torrc && \
-    echo "DataDirectory /var/lib/tor" >> /etc/tor/torrc
+WORKDIR /app
 
-# Set permissions for Tor, app directory, and nginx
-RUN mkdir -p /var/lib/tor && \
-    chown -R user:user /var/lib/tor && \
-    chmod 700 /var/lib/tor && \
-    chown -R user:user /app && \
-    mkdir -p /var/log/nginx /var/lib/nginx /tmp && \
-    chown -R user:user /var/log/nginx /var/lib/nginx /tmp
+# Copy dependency file first for better layer caching
+COPY pyproject.toml ./
 
-# Pre-create streamlit config dir in home
-RUN mkdir -p /home/user/.streamlit && chown -R user:user /home/user
+# Install dependencies only
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --no-install-project --all-extras --compile-bytecode
 
-# Copy the rest of the application
-COPY --chown=user:user . .
+# Copy source code
+COPY . .
 
-# Install Scrapling in editable mode and its browser dependencies
-RUN uv pip install --system -e ./Scrapling[fetchers]
-RUN playwright install chromium
+# Install browsers and project in one optimized layer
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \
+    apt-get update && \
+    uv run playwright install-deps chromium && \
+    uv run playwright install chromium && \
+    uv sync --all-extras --compile-bytecode && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-# Set permissions for the start script
-RUN chmod +x start.sh
+# Create a non-root user
+RUN useradd -m -u 1000 user && \
+    chown -R user:user /app
 
-# Switch to non-root user
+# Switch to the non-root user
 USER user
-ENV PATH="/home/user/.local/bin:$PATH"
 
-# Expose port
+# Expose port for MCP server HTTP transport
 EXPOSE 7860
 
-# Set the entrypoint
-ENTRYPOINT ["./start.sh"]
+# Set entrypoint to run scrapling
+ENTRYPOINT ["uv", "run", "scrapling"]
+
+# Default command (can be overridden)
+CMD ["mcp", "--http", "--port", "7860", "--host", "0.0.0.0"]

LICENSE

@@ -1,21 +1,28 @@
-MIT License
+BSD 3-Clause License
 
-Copyright (c) 2024 Owen Singh
+Copyright (c) 2024, Karim shoair
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
 
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

MANIFEST.in

@@ -0,0 +1,12 @@
+include LICENSE
+include *.db
+include *.js
+include scrapling/*.db
+include scrapling/*.db*
+include scrapling/*.db-*
+include scrapling/py.typed
+include scrapling/.scrapling_dependencies_installed
+include .scrapling_dependencies_installed
+
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]

README.md

@@ -1,406 +1,437 @@
 ---
 title: Scraper Hub
-emoji: 🌐
-colorFrom: blue
-colorTo: red
+emoji: 🕷️
+colorFrom: purple
+colorTo: blue
 sdk: docker
 app_port: 7860
 ---
-
-# 🌐 CyberScraper 2077
+<!-- mcp-name: io.github.D4Vinci/Scrapling -->
+
+<h1 align="center">
+    <a href="https://scrapling.readthedocs.io">
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_dark.svg?sanitize=true">
+          <img alt="Scrapling Poster" src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_light.svg?sanitize=true">
+        </picture>
+    </a>
+    <br>
+    <small>Effortless Web Scraping for the Modern Web</small>
+</h1>
 
 <p align="center">
-  <img src="https://i.postimg.cc/j5b7QSzg/scraper.png" alt="CyberScraper 2077 Logo">
+    <a href="https://trendshift.io/repositories/14244" target="_blank"><img src="https://trendshift.io/api/badge/repositories/14244" alt="D4Vinci%2FScrapling | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+    <br/>
+    <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_AR.md">العربيه</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_ES.md">Español</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_DE.md">Deutsch</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_CN.md">简体中文</a> | <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_JP.md">日本語</a> |  <a href="https://github.com/D4Vinci/Scrapling/blob/main/docs/README_RU.md">Русский</a>
+    <br/>
+    <a href="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml" alt="Tests">
+        <img alt="Tests" src="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg"></a>
+    <a href="https://badge.fury.io/py/Scrapling" alt="PyPI version">
+        <img alt="PyPI version" src="https://badge.fury.io/py/Scrapling.svg"></a>
+    <a href="https://pepy.tech/project/scrapling" alt="PyPI Downloads">
+        <img alt="PyPI Downloads" src="https://static.pepy.tech/personalized-badge/scrapling?period=total&units=INTERNATIONAL_SYSTEM&left_color=GREY&right_color=GREEN&left_text=Downloads"></a>
+    <br/>
+    <a href="https://discord.gg/EMgGbDceNQ" alt="Discord" target="_blank">
+      <img alt="Discord" src="https://img.shields.io/discord/1360786381042880532?style=social&logo=discord&link=https%3A%2F%2Fdiscord.gg%2FEMgGbDceNQ">
+    </a>
+    <a href="https://x.com/Scrapling_dev" alt="X (formerly Twitter)">
+      <img alt="X (formerly Twitter) Follow" src="https://img.shields.io/twitter/follow/Scrapling_dev?style=social&logo=x&link=https%3A%2F%2Fx.com%2FScrapling_dev">
+    </a>
+    <br/>
+    <a href="https://pypi.org/project/scrapling/" alt="Supported Python versions">
+        <img alt="Supported Python versions" src="https://img.shields.io/pypi/pyversions/scrapling.svg"></a>
 </p>
 
 <p align="center">
-  <img src="https://i.postimg.cc/9MKqtn2g/68747470733a2f2f692e706f7374696d672e63632f74346d64347a74762f6379626572736372617065722d323037372e6a70.jpg">
+    <a href="https://scrapling.readthedocs.io/en/latest/parsing/selection/"><strong>Selection methods</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/fetching/choosing/"><strong>Fetchers</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/architecture.html"><strong>Spiders</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/proxy-blocking.html"><strong>Proxy Rotation</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/cli/overview/"><strong>CLI</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/ai/mcp-server/"><strong>MCP</strong></a>
 </p>
 
-[![Python](https://img.shields.io/badge/Python-blue)](https://www.python.org/downloads/)
-[![Streamlit](https://img.shields.io/badge/Streamlit-FF4B4B)](https://streamlit.io/)
-[![License](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
-[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](http://makeapullrequest.com)
-
-> Rip data from the net, leaving no trace. Welcome to the future of web scraping.
-
-## 🔍 About
-
-CyberScraper 2077 is not just another web scraping tool – it's a glimpse into the future of data extraction. Born from the neon-lit streets of a cyberpunk world, this AI-powered scraper uses OpenAI, Gemini and LocalLLM Models to slice through the web's defenses, extracting the data you need with unparalleled precision and style.
-
-Whether you're a corpo data analyst, a street-smart netrunner, or just someone looking to pull information from the digital realm, CyberScraper 2077 has got you covered.
-
-<p align="center">
-  <img src="https://i.postimg.cc/3NHb15wq/20240821-074556.gif">
-</p>
-
-## ✨ Features
-
-- **AI-Powered Extraction**: Utilizes cutting-edge AI models to understand and parse web content intelligently.
-- **Sleek Streamlit Interface**: User-friendly GUI that even a chrome-armed street samurai could navigate.
-- **Multi-Format Support**: Export your data in JSON, CSV, HTML, SQL or Excel – whatever fits your cyberdeck.
-- **Tor Network Support**: Safely scrape .onion sites through the Tor network with automatic routing and security features.
-- **Stealth Mode**: Implemented stealth mode parameters that help avoid detection as a bot.
-- **Ollama Support**: Use a huge library of open source LLMs.
-- **Async Operations**: Lightning-fast scraping that would make a Trauma Team jealous.
-- **Smart Parsing**: Structures scraped content as if it was extracted straight from the engram of a master netrunner.
-- **Caching**: Implemented content-based and query-based caching using LRU cache and a custom dictionary to reduce redundant API calls.
-- **Upload to Google Sheets**: Now you can easily upload your extracted CSV data to Google Sheets with one click.
-- **Bypass Captcha**: Bypass captcha by using the -captcha at the end of the URL. (Currently only works natively, doesn't work on Docker)
-- **Current Browser**: The current browser feature uses your local browser instance which will help you bypass 99% of bot detections. (Only use when necessary)
-- **Navigate through the Pages (BETA)**: Navigate through the webpage and scrape data from different pages.
-
-## 🪟 For Windows Users
+Scrapling is an adaptive Web Scraping framework that handles everything from a single request to a full-scale crawl.
 
-Please follow the Docker Container Guide given below, as I won't be able to maintain another version for Windows systems.
+Its parser learns from website changes and automatically relocates your elements when pages update. Its fetchers bypass anti-bot systems like Cloudflare Turnstile out of the box. And its spider framework lets you scale up to concurrent, multi-session crawls with pause/resume and automatic proxy rotation — all in a few lines of Python. One library, zero compromises.
 
-## 🛠 Installation
+Blazing fast crawls with real-time stats and streaming. Built by Web Scrapers for Web Scrapers and regular users, there's something for everyone.
 
-**Note: CyberScraper 2077 requires Python 3.10 or higher.**
-
-1. Clone this repository:
-   ```bash
-   git clone https://github.com/itsOwen/CyberScraper-2077.git
-   cd CyberScraper-2077
-   ```
-
-2. Create and activate a virtual environment:
-   ```bash
-   virtualenv venv
-   source venv/bin/activate  # Optional
-   ```
-
-3. Install the required packages:
-   ```bash
-   pip install -r requirements.txt
-   ```
-
-4. Install the playwright:
-   ```bash
-   playwright install
-   ```
-
-5. Set OpenAI & Gemini Key in your environment:
-
-   Linux/Mac:
-   ```bash
-   export OPENAI_API_KEY="your-api-key-here"
-   export GOOGLE_API_KEY="your-api-key-here"
-   ```
-
-### Using Ollama
-
-Note: I only recommend using OpenAI and Gemini API as these models are really good at following instructions. If you are using open-source LLMs, make sure you have a good system as the speed of the data generation/presentation depends on how well your system can run the LLM. You may also have to fine-tune the prompt and add some additional filters yourself.
-
-```bash
-1. Setup Ollama using `pip install ollama`
-2. Download Ollama from the official website: https://ollama.com/download
-3. Now type: ollama pull llama3.1 or whatever LLM you want to use.
-4. Now follow the rest of the steps below.
+```python
+from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
+StealthyFetcher.adaptive = True
+p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True)  # Fetch website under the radar!
+products = p.css('.product', auto_save=True)                                        # Scrape data that survives website design changes!
+products = p.css('.product', adaptive=True)                                         # Later, if the website structure changes, pass `adaptive=True` to find them!
 ```
+Or scale up to full crawls
+```python
+from scrapling.spiders import Spider, Response
 
-## 🐳 Docker Installation
-
-If you prefer to use Docker, follow these steps to set up and run CyberScraper 2077:
-
-1. Ensure you have Docker installed on your system.
-
-2. Clone this repository:
-   ```bash
-   git clone https://github.com/itsOwen/CyberScraper-2077.git
-   cd CyberScraper-2077
-   ```
+class MySpider(Spider):
+  name = "demo"
+  start_urls = ["https://example.com/"]
 
-3. Build the Docker image:
-   ```bash
-   docker build -t cyberscraper-2077 .
-   ```
+  async def parse(self, response: Response):
+      for item in response.css('.product'):
+          yield {"title": item.css('h2::text').get()}
 
-4. Run the container:
-   ```bash
-   docker run -p 8501:8501 -e OPENAI_API_KEY="your-actual-api-key" -e GOOGLE_API_KEY="your-actual-api-key" cyberscraper-2077
-   ```
+MySpider().start()
+```
 
-### Using Ollama with Docker
+# Platinum Sponsors
 
-If you want to use Ollama with the Docker setup:
+<i><sub>Do you want to be the first company to show up here? Click [here](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
+# Sponsors 
 
-1. Install Ollama on your host machine following the instructions at https://ollama.com/download
+<!-- sponsors -->
 
-2. Run Ollama on your host machine:
-   ```bash
-   ollama pull llama3.1
-   ```
+<a href="https://www.scrapeless.com/en?utm_source=official&utm_term=scrapling" target="_blank" title="Effortless Web Scraping Toolkit for Business and Developers"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/scrapeless.jpg"></a>
+<a href="https://www.thordata.com/?ls=github&lk=github" target="_blank" title="Unblockable proxies and scraping infrastructure, delivering real-time, reliable web data to power AI models and workflows."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/thordata.jpg"></a>
+<a href="https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling" target="_blank" title="Evomi is your Swiss Quality Proxy Provider, starting at $0.49/GB"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/evomi.png"></a>
+<a href="https://serpapi.com/?utm_source=scrapling" target="_blank" title="Scrape Google and other search engines with SerpApi"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SerpApi.png"></a>
+<a href="https://visit.decodo.com/Dy6W0b" target="_blank" title="Try the Most Efficient Residential Proxies for Free"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/decodo.png"></a>
+<a href="https://petrosky.io/d4vinci" target="_blank" title="PetroSky delivers cutting-edge VPS hosting."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/petrosky.png"></a>
+<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
+<a href="https://proxyempire.io/" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
+<a href="https://hypersolutions.co/?utm_source=github&utm_medium=readme&utm_campaign=scrapling" target="_blank" title="Bot Protection Bypass API for Akamai, DataDome, Incapsula & Kasada"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/HyperSolutions.png"></a>
 
-3. Find your host machine's IP address:
-   - On Linux/Mac: `ifconfig` or `ip addr show`
-   - On Windows: `ipconfig`
 
-4. Run the Docker container with the host network and set the Ollama URL:
-   ```bash
-   docker run -e OLLAMA_BASE_URL=http://host.docker.internal:11434 -p 8501:8501 cyberscraper-2077
-   ```
+<a href="https://www.swiftproxy.net/" target="_blank" title="Unlock Reliable Proxy Services with Swiftproxy!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/swiftproxy.png"></a>
+<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
+<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
 
-   Now visit the url: http://localhost:8501/
+<!-- /sponsors -->
 
-   On Linux you might need to use this below:
-   ```bash
-   docker run -e OLLAMA_BASE_URL=http://<your-host-ip>:11434 -p 8501:8501 cyberscraper-2077
-   ```
-   Replace `<your-host-ip>` with your actual host machine IP address.
+<i><sub>Do you want to show your ad here? Click [here](https://github.com/sponsors/D4Vinci) and choose the tier that suites you!</sub></i>
 
-5. In the Streamlit interface, select the Ollama model you want to use (e.g., "ollama:llama3.1").
+---
 
-Note: Ensure that your firewall allows connections to port 11434 for Ollama.
+## Key Features
+
+### Spiders — A Full Crawling Framework
+- 🕷️ **Scrapy-like Spider API**: Define spiders with `start_urls`, async `parse` callbacks, and `Request`/`Response` objects.
+- ⚡ **Concurrent Crawling**: Configurable concurrency limits, per-domain throttling, and download delays.
+- 🔄 **Multi-Session Support**: Unified interface for HTTP requests, and stealthy headless browsers in a single spider — route requests to different sessions by ID.
+- 💾 **Pause & Resume**: Checkpoint-based crawl persistence. Press Ctrl+C for a graceful shutdown; restart to resume from where you left off.
+- 📡 **Streaming Mode**: Stream scraped items as they arrive via `async for item in spider.stream()` with real-time stats — ideal for UI, pipelines, and long-running crawls.
+- 🛡️ **Blocked Request Detection**: Automatic detection and retry of blocked requests with customizable logic.
+- 📦 **Built-in Export**: Export results through hooks and your own pipeline or the built-in JSON/JSONL with `result.items.to_json()` / `result.items.to_jsonl()` respectively.
+
+### Advanced Websites Fetching with Session Support
+- **HTTP Requests**: Fast and stealthy HTTP requests with the `Fetcher` class. Can impersonate browsers' TLS fingerprint, headers, and use HTTP/3.
+- **Dynamic Loading**: Fetch dynamic websites with full browser automation through the `DynamicFetcher` class supporting Playwright's Chromium and Google's Chrome.
+- **Anti-bot Bypass**: Advanced stealth capabilities with `StealthyFetcher` and fingerprint spoofing. Can easily bypass all types of Cloudflare's Turnstile/Interstitial with automation.
+- **Session Management**: Persistent session support with `FetcherSession`, `StealthySession`, and `DynamicSession` classes for cookie and state management across requests.
+- **Proxy Rotation**: Built-in `ProxyRotator` with cyclic or custom rotation strategies across all session types, plus per-request proxy overrides.
+- **Domain Blocking**: Block requests to specific domains (and their subdomains) in browser-based fetchers.
+- **Async Support**: Complete async support across all fetchers and dedicated async session classes.
+
+### Adaptive Scraping & AI Integration
+- 🔄 **Smart Element Tracking**: Relocate elements after website changes using intelligent similarity algorithms.
+- 🎯 **Smart Flexible Selection**: CSS selectors, XPath selectors, filter-based search, text search, regex search, and more.
+- 🔍 **Find Similar Elements**: Automatically locate elements similar to found elements.
+- 🤖 **MCP Server to be used with AI**: Built-in MCP server for AI-assisted Web Scraping and data extraction. The MCP server features powerful, custom capabilities that leverage Scrapling to extract targeted content before passing it to the AI (Claude/Cursor/etc), thereby speeding up operations and reducing costs by minimizing token usage. ([demo video](https://www.youtube.com/watch?v=qyFk3ZNwOxE))
+
+### High-Performance & battle-tested Architecture
+- 🚀 **Lightning Fast**: Optimized performance outperforming most Python scraping libraries.
+- 🔋 **Memory Efficient**: Optimized data structures and lazy loading for a minimal memory footprint.
+- ⚡ **Fast JSON Serialization**: 10x faster than the standard library.
+- 🏗️ **Battle tested**: Not only does Scrapling have 92% test coverage and full type hints coverage, but it has been used daily by hundreds of Web Scrapers over the past year.
+
+### Developer/Web Scraper Friendly Experience
+- 🎯 **Interactive Web Scraping Shell**: Optional built-in IPython shell with Scrapling integration, shortcuts, and new tools to speed up Web Scraping scripts development, like converting curl requests to Scrapling requests and viewing requests results in your browser.
+- 🚀 **Use it directly from the Terminal**: Optionally, you can use Scrapling to scrape a URL without writing a single line of code!
+- 🛠️ **Rich Navigation API**: Advanced DOM traversal with parent, sibling, and child navigation methods.
+- 🧬 **Enhanced Text Processing**: Built-in regex, cleaning methods, and optimized string operations.
+- 📝 **Auto Selector Generation**: Generate robust CSS/XPath selectors for any element.
+- 🔌 **Familiar API**: Similar to Scrapy/BeautifulSoup with the same pseudo-elements used in Scrapy/Parsel.
+- 📘 **Complete Type Coverage**: Full type hints for excellent IDE support and code completion. The entire codebase is automatically scanned with **PyRight** and **MyPy** with each change.
+- 🔋 **Ready Docker image**: With each release, a Docker image containing all browsers is automatically built and pushed.
+
+## Getting Started
+
+Let's give you a quick glimpse of what Scrapling can do without deep diving.
+
+### Basic Usage
+HTTP requests with session support
+```python
+from scrapling.fetchers import Fetcher, FetcherSession
 
-## 🚀 Usage
+with FetcherSession(impersonate='chrome') as session:  # Use latest version of Chrome's TLS fingerprint
+    page = session.get('https://quotes.toscrape.com/', stealthy_headers=True)
+    quotes = page.css('.quote .text::text').getall()
 
-1. Fire up the Streamlit app:
-   ```bash
-   streamlit run main.py
-   ```
+# Or use one-off requests
+page = Fetcher.get('https://quotes.toscrape.com/')
+quotes = page.css('.quote .text::text').getall()
+```
+Advanced stealth mode
+```python
+from scrapling.fetchers import StealthyFetcher, StealthySession
 
-2. Open your browser and navigate to `http://localhost:8501`.
+with StealthySession(headless=True, solve_cloudflare=True) as session:  # Keep the browser open until you finish
+    page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False)
+    data = page.css('#padded_content a').getall()
 
-3. Enter the URL of the site you want to scrape or ask a question about the data you need.
+# Or use one-off request style, it opens the browser for this request, then closes it after finishing
+page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare')
+data = page.css('#padded_content a').getall()
+```
+Full browser automation
+```python
+from scrapling.fetchers import DynamicFetcher, DynamicSession
 
-4. Ask the chatbot to extract the data in any format. Select whatever data you want to export or even everything from the webpage.
+with DynamicSession(headless=True, disable_resources=False, network_idle=True) as session:  # Keep the browser open until you finish
+    page = session.fetch('https://quotes.toscrape.com/', load_dom=False)
+    data = page.xpath('//span[@class="text"]/text()').getall()  # XPath selector if you prefer it
 
-5. Watch as CyberScraper 2077 tears through the net, extracting your data faster than you can say "flatline"!
+# Or use one-off request style, it opens the browser for this request, then closes it after finishing
+page = DynamicFetcher.fetch('https://quotes.toscrape.com/')
+data = page.css('.quote .text::text').getall()
+```
 
-Example usage with page ranges:
+### Spiders
+Build full crawlers with concurrent requests, multiple session types, and pause/resume:
+```python
+from scrapling.spiders import Spider, Request, Response
+
+class QuotesSpider(Spider):
+    name = "quotes"
+    start_urls = ["https://quotes.toscrape.com/"]
+    concurrent_requests = 10
+    
+    async def parse(self, response: Response):
+        for quote in response.css('.quote'):
+            yield {
+                "text": quote.css('.text::text').get(),
+                "author": quote.css('.author::text').get(),
+            }
+            
+        next_page = response.css('.next a')
+        if next_page:
+            yield response.follow(next_page[0].attrib['href'])
+
+result = QuotesSpider().start()
+print(f"Scraped {len(result.items)} quotes")
+result.items.to_json("quotes.json")
 ```
-https://example.com/products 1-5
-https://example.com/search?q=cyberpunk&page={page} 1-10
+Use multiple session types in a single spider:
+```python
+from scrapling.spiders import Spider, Request, Response
+from scrapling.fetchers import FetcherSession, AsyncStealthySession
+
+class MultiSessionSpider(Spider):
+    name = "multi"
+    start_urls = ["https://example.com/"]
+    
+    def configure_sessions(self, manager):
+        manager.add("fast", FetcherSession(impersonate="chrome"))
+        manager.add("stealth", AsyncStealthySession(headless=True), lazy=True)
+    
+    async def parse(self, response: Response):
+        for link in response.css('a::attr(href)').getall():
+            # Route protected pages through the stealth session
+            if "protected" in link:
+                yield Request(link, sid="stealth")
+            else:
+                yield Request(link, sid="fast", callback=self.parse)  # explicit callback
 ```
-
-## 🌐 Multi-Page Scraping (BETA)
-
-> **Note**: The multi-page scraping feature is currently in beta. While functional, you may encounter occasional issues or unexpected behavior. We appreciate your feedback and patience as we continue to improve this feature.
-
-CyberScraper 2077 now supports multi-page scraping, allowing you to extract data from multiple pages of a website in one go. This feature is perfect for scraping paginated content, search results, or any site with data spread across multiple pages.
-
-### How to Use Multi-Page Scraping
-
-I suggest you enter the URL structure every time if you want to scrape multiple pages so it can detect the URL structure easily. It detects nearly all URL types.
-
-1. **Basic Usage**:
-   To scrape multiple pages, use the following format when entering the URL:
-   ```
-   https://example.com/page 1-5
-   https://example.com/p/ 1-6
-   https://example.com/xample/something-something-1279?p=1 1-3
-   ```
-   This will scrape pages 1 through 5 of the website.
-
-2. **Custom Page Ranges**:
-   You can specify custom page ranges:
-   ```
-   https://example.com/p/ 1-5,7,9-12
-   https://example.com/xample/something-something-1279?p=1 1,7,8,9
-   ```
-   This will scrape pages 1 to 5, page 7, and pages 9 to 12.
-
-3. **URL Patterns**:
-   For websites with different URL structures, you can specify a pattern:
-   ```
-   https://example.com/search?q=cyberpunk&page={page} 1-5
-   ```
-   Replace `{page}` with where the page number should be in the URL.
-
-4. **Automatic Pattern Detection**:
-   If you don't specify a pattern, CyberScraper 2077 will attempt to detect the URL pattern automatically. However, for best results, specifying the pattern is recommended.
-
-### Tips for Effective Multi-Page Scraping
-
-- Start with a small range of pages to test before scraping a large number.
-- Be mindful of the website's load and your scraping speed to avoid overloading servers.
-- Use the `simulate_human` option for more natural scraping behavior on sites with anti-bot measures.
-- Regularly check the website's `robots.txt` file and terms of service to ensure compliance.
-
-### Example
-
-```bash
-URL Example : "https://news.ycombinator.com/?p=1 1-3 or 1,2,3,4"
+Pause and resume long crawls with checkpoints by running the spider like this:
+```python
+QuotesSpider(crawldir="./crawl_data").start()
 ```
+Press Ctrl+C to pause gracefully — progress is saved automatically. Later, when you start the spider again, pass the same `crawldir`, and it will resume from where it stopped.
 
-If you want to scrape a specific page, just enter the query "please scrape page number 1 or 2". If you want to scrape all pages, simply give a query like "scrape all pages in csv" or whatever format you want.
-
-## 🧅 Tor Network Scraping
-
-> **Note**: The Tor network scraping feature allows you to access and scrape .onion sites. This feature requires additional setup and should be used responsibly and legally.
-
-CyberScraper 2077 now supports scraping .onion sites through the Tor network, allowing you to access and extract data from the dark web safely and anonymously. This feature is perfect for researchers, security analysts, and investigators who need to gather information from Tor hidden services.
-
-### Prerequisites
-
-1. Install Tor on your system:
-   ```bash
-   # Ubuntu/Debian
-   sudo apt install tor
-
-   # macOS (using Homebrew)
-   brew install tor
-
-   # Start the Tor service
-   sudo service tor start  # on Linux
-   brew services start tor # on macOS
-   ```
-
-2. Install additional Python packages:
-   ```bash
-   pip install PySocks requests[socks]
-   ```
-
-### Using Tor Scraping
-
-1. **Basic Usage**:
-   Simply enter an .onion URL, and CyberScraper will automatically detect and route it through the Tor network:
-   ```
-   http://example123abc.onion
-   ```
-
-2. **Safety Features**:
-   - Automatic .onion URL detection
-   - Built-in connection verification
-   - Tor Browser-like request headers
-   - Automatic circuit isolation
+### Advanced Parsing & Navigation
+```python
+from scrapling.fetchers import Fetcher
+
+# Rich element selection and navigation
+page = Fetcher.get('https://quotes.toscrape.com/')
+
+# Get quotes with multiple selection methods
+quotes = page.css('.quote')  # CSS selector
+quotes = page.xpath('//div[@class="quote"]')  # XPath
+quotes = page.find_all('div', {'class': 'quote'})  # BeautifulSoup-style
+# Same as
+quotes = page.find_all('div', class_='quote')
+quotes = page.find_all(['div'], class_='quote')
+quotes = page.find_all(class_='quote')  # and so on...
+# Find element by text content
+quotes = page.find_by_text('quote', tag='div')
+
+# Advanced navigation
+quote_text = page.css('.quote')[0].css('.text::text').get()
+quote_text = page.css('.quote').css('.text::text').getall()  # Chained selectors
+first_quote = page.css('.quote')[0]
+author = first_quote.next_sibling.css('.author::text')
+parent_container = first_quote.parent
+
+# Element relationships and similarity
+similar_elements = first_quote.find_similar()
+below_elements = first_quote.below_elements()
+```
+You can use the parser right away if you don't want to fetch websites like below:
+```python
+from scrapling.parser import Selector
 
-### Configuration Options
+page = Selector("<html>...</html>")
+```
+And it works precisely the same way!
 
-You can customize the Tor scraping behavior by adjusting the following settings:
+### Async Session Management Examples
 ```python
-tor_config = TorConfig(
-    socks_port=9050,          # Default Tor SOCKS port
-    circuit_timeout=10,        # Timeout for circuit creation
-    auto_renew_circuit=True,   # Automatically renew Tor circuit
-    verify_connection=True     # Verify Tor connection before scraping
-)
+import asyncio
+from scrapling.fetchers import FetcherSession, AsyncStealthySession, AsyncDynamicSession
+
+async with FetcherSession(http3=True) as session:  # `FetcherSession` is context-aware and can work in both sync/async patterns
+    page1 = session.get('https://quotes.toscrape.com/')
+    page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135')
+
+# Async session usage
+async with AsyncStealthySession(max_pages=2) as session:
+    tasks = []
+    urls = ['https://example.com/page1', 'https://example.com/page2']
+    
+    for url in urls:
+        task = session.fetch(url)
+        tasks.append(task)
+    
+    print(session.get_pool_stats())  # Optional - The status of the browser tabs pool (busy/free/error)
+    results = await asyncio.gather(*tasks)
+    print(session.get_pool_stats())
 ```
 
-### Security Considerations
+## CLI & Interactive Shell
 
-- Always ensure you're complying with local laws and regulations
-- Use a VPN in addition to Tor for extra security
-- Be patient as Tor connections can be slower than regular web scraping
-- Avoid sending personal or identifying information through Tor
-- Some .onion sites may be offline or unreachable
+Scrapling includes a powerful command-line interface:
 
-### Docker Support
+[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339)
 
-For Docker users, add these additional flags to enable Tor support:
+Launch the interactive Web Scraping shell
 ```bash
-docker run -p 8501:8501 \
-  --network="host" \
-  -e OPENAI_API_KEY="your-api-key" \
-  cyberscraper-2077
+scrapling shell
 ```
-
-### Example Usage
-
-<p align="center">
-  <img src="https://i.postimg.cc/3JvhgtMP/cyberscraper-onion.png" alt="CyberScraper 2077 Onion Scrape">
-</p>
-
-## 🔐 Setup Google Sheets Authentication
-
-1. Go to the Google Cloud Console (https://console.cloud.google.com/).
-2. Select your project.
-3. Navigate to "APIs & Services" > "Credentials".
-4. Find your existing OAuth 2.0 Client ID and delete it.
-5. Click "Create Credentials" > "OAuth client ID".
-6. Choose "Web application" as the application type.
-7. Name your client (e.g., "CyberScraper 2077 Web Client").
-8. Under "Authorized JavaScript origins", add:
-   - http://localhost:8501
-   - http://localhost:8502
-   - http://127.0.0.1:8501
-   - http://127.0.0.1:8502
-9. Under "Authorized redirect URIs", add:
-   - http://localhost:8501/
-   - http://127.0.0.1:8501/
-   - http://localhost:8502/
-   - http://127.0.0.1:8502/
-10. Click "Create" to generate the new client ID.
-11. Download the new client configuration JSON file and rename it to `client_secret.json`.
-
-## ⚙️ Adjusting PlaywrightScraper Settings (optional)
-
-Customize the `PlaywrightScraper` settings to fit your scraping needs. If some websites are giving you issues, you might want to check the behavior of the website:
-
+Extract pages to a file directly without programming (Extracts the content inside the `body` tag by default). If the output file ends with `.txt`, then the text content of the target will be extracted. If it ends in `.md`, it will be a Markdown representation of the HTML content; if it ends in `.html`, it will be the HTML content itself.
 ```bash
-use_stealth: bool = True,
-simulate_human: bool = False,
-use_custom_headers: bool = True,
-hide_webdriver: bool = True,
-bypass_cloudflare: bool = True:
+scrapling extract get 'https://example.com' content.md
+scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome'  # All elements matching the CSS selector '#fromSkipToProducts'
+scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless
+scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare
 ```
 
-Adjust these settings based on your target website and environment for optimal results.
+> [!NOTE]
+> There are many additional features, but we want to keep this page concise, including the MCP server and the interactive Web Scraping Shell. Check out the full documentation [here](https://scrapling.readthedocs.io/en/latest/)
 
-You can also bypass the captcha using the ```-captcha``` parameter at the end of the URL. The browser window will pop up, complete the captcha, and go back to your terminal window. Press enter and the bot will complete its task.
+## Performance Benchmarks
 
-## 🤝 Contributing
+Scrapling isn't just powerful—it's also blazing fast. The following benchmarks compare Scrapling's parser with the latest versions of other popular libraries.
 
-We welcome all cyberpunks, netrunners, and code samurais to contribute to CyberScraper 2077!
+### Text Extraction Speed Test (5000 nested elements)
 
-1. Fork the repository
-2. Create your feature branch (`git checkout -b feature/amazing-feature`)
-3. Commit your changes (`git commit -m 'Add some amazing feature'`)
-4. Push to the branch (`git push origin feature/amazing-feature`)
-5. Open a Pull Request
+| # |      Library      | Time (ms) | vs Scrapling | 
+|---|:-----------------:|:---------:|:------------:|
+| 1 |     Scrapling     |   2.02    |     1.0x     |
+| 2 |   Parsel/Scrapy   |   2.04    |     1.01     |
+| 3 |     Raw Lxml      |   2.54    |    1.257     |
+| 4 |      PyQuery      |   24.17   |     ~12x     |
+| 5 |    Selectolax     |   82.63   |     ~41x     |
+| 6 |  MechanicalSoup   |  1549.71  |   ~767.1x    |
+| 7 |   BS4 with Lxml   |  1584.31  |   ~784.3x    |
+| 8 | BS4 with html5lib |  3391.91  |   ~1679.1x   |
 
-## 🔧 Troubleshooting
 
-Ran into a glitch in the matrix? Let me know by adding the issue to this repo so that we can fix it together.
+### Element Similarity & Text Search Performance
 
-## ❓ FAQ
+Scrapling's adaptive element finding capabilities significantly outperform alternatives:
 
-**Q: Is CyberScraper 2077 legal to use?**
-A: CyberScraper 2077 is designed for ethical web scraping. Always ensure you have the right to scrape a website and respect their robots.txt file.
+| Library     | Time (ms) | vs Scrapling |
+|-------------|:---------:|:------------:|
+| Scrapling   |   2.39    |     1.0x     |
+| AutoScraper |   12.45   |    5.209x    |
 
-**Q: Can I use this for commercial purposes?**
-A: Yes, under the terms of the MIT License.
 
-## 📄 License
+> All benchmarks represent averages of 100+ runs. See [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) for methodology.
 
-This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. Use it, mod it, sell it – just don't blame us if you end up flatlined.
+## Installation
 
-## 📞 Contact
+Scrapling requires Python 3.10 or higher:
 
-Got questions? Need support? Want to hire me for a gig?
-
-- Email: owensingh72@proton.me
-- Website: [owen.sh](https://owen.sh)
-
-## 🚨 Disclaimer
-
-Listen up, choombas! Before you jack into this code, you better understand the risks:
+```bash
+pip install scrapling
+```
 
-1. This software is provided "as is", without warranty of any kind, express or implied.
+This installation only includes the parser engine and its dependencies, without any fetchers or commandline dependencies.
+
+### Optional Dependencies
+
+1. If you are going to use any of the extra features below, the fetchers, or their classes, you will need to install fetchers' dependencies and their browser dependencies as follows:
+    ```bash
+    pip install "scrapling[fetchers]"
+    
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
+    ```
+
+    This downloads all browsers, along with their system dependencies and fingerprint manipulation dependencies.
+
+    Or you can install them from the code instead of running a command like this:
+    ```python
+    from scrapling.cli import install
+    
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
+2. Extra features:
+   - Install the MCP server feature:
+       ```bash
+       pip install "scrapling[ai]"
+       ```
+   - Install shell features (Web Scraping shell and the `extract` command): 
+       ```bash
+       pip install "scrapling[shell]"
+       ```
+   - Install everything: 
+       ```bash
+       pip install "scrapling[all]"
+       ```
+   Remember that you need to install the browser dependencies with `scrapling install` after any of these extras (if you didn't already)
+
+### Docker
+You can also install a Docker image with all extras and browsers with the following command from DockerHub:
+```bash
+docker pull pyd4vinci/scrapling
+```
+Or download it from the GitHub registry:
+```bash
+docker pull ghcr.io/d4vinci/scrapling:latest
+```
+This image is automatically built and pushed using GitHub Actions and the repository's main branch.
 
-2. The authors are not liable for any damages or losses resulting from the use of this software.
+## Contributing
 
-3. This tool is intended for educational and research purposes only. Any illegal use is strictly prohibited.
+We welcome contributions! Please read our [contributing guidelines](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md) before getting started.
 
-4. We do not guarantee the accuracy, completeness, or reliability of any data obtained through this tool.
+## Disclaimer
 
-5. By using this software, you acknowledge that you are doing so at your own risk.
+> [!CAUTION]
+> This library is provided for educational and research purposes only. By using this library, you agree to comply with local and international data scraping and privacy laws. The authors and contributors are not responsible for any misuse of this software. Always respect the terms of service of websites and robots.txt files.
 
-6. You are responsible for complying with all applicable laws and regulations in your use of this software.
+## License
 
-7. We reserve the right to modify or discontinue the software at any time without notice.
+This work is licensed under the BSD-3-Clause License.
 
-Remember, samurai: In the dark future of the NET, knowledge is power, but it's also a double-edged sword. Use this tool wisely, and may your connection always be strong and your firewalls impenetrable. Stay frosty out there in the digital frontier.
+## Acknowledgments
 
-![Alt](https://repobeats.axiom.co/api/embed/80758496e19179f355d6d71c180db7aca66d396b.svg "Repobeats analytics image")
+This project includes code adapted from:
+- Parsel (BSD License)—Used for [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py) submodule
 
 ---
-
-<p align="center">
-  <strong>CyberScraper 2077 – Because in 2077, what makes someone a criminal? Getting caught.</strong>
-</p>
-
-<p align="center">
-  Built with love and chrome by the streets of Night City | © 2077 Owen Singh
-</p>
+<div align="center"><small>Designed & crafted with ❤️ by Karim Shoair.</small></div><br>

ROADMAP.md

@@ -0,0 +1,14 @@
+## TODOs
+- [x] Add more tests and increase the code coverage.
+- [x] Structure the tests folder in a better way.
+- [x] Add more documentation.
+- [x] Add the browsing ability.
+- [x] Create detailed documentation for the 'readthedocs' website, preferably add GitHub action for deploying it.
+- [ ] Create a Scrapy plugin/decorator to make it replace parsel in the response argument when needed.
+- [x] Need to add more functionality to `AttributesHandler` and more navigation functions to `Selector` object (ex: functions similar to map, filter, and reduce functions but here pass it to the element and the function is executed on children, siblings, next elements, etc...)
+- [x] Add `.filter` method to `Selectors` object and other similar methods.
+- [ ] Add functionality to automatically detect pagination URLs
+- [ ] Add the ability to auto-detect schemas in pages and manipulate them.
+- [ ] Add `analyzer` ability that tries to learn about the page through meta-elements and return what it learned
+- [ ] Add the ability to generate a regex from a group of elements (Like for all href attributes)
+- 

benchmarks.py

@@ -0,0 +1,146 @@
+import functools
+import time
+import timeit
+from statistics import mean
+
+import requests
+from autoscraper import AutoScraper
+from bs4 import BeautifulSoup
+from lxml import etree, html
+from mechanicalsoup import StatefulBrowser
+from parsel import Selector
+from pyquery import PyQuery as pq
+from selectolax.parser import HTMLParser
+
+from scrapling import Selector as ScraplingSelector
+
+large_html = (
+    "<html><body>" + '<div class="item">' * 5000 + "</div>" * 5000 + "</body></html>"
+)
+
+
+def benchmark(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        benchmark_name = func.__name__.replace("test_", "").replace("_", " ")
+        print(f"-> {benchmark_name}", end=" ", flush=True)
+        # Warm-up phase
+        timeit.repeat(
+            lambda: func(*args, **kwargs), number=2, repeat=2, globals=globals()
+        )
+        # Measure time (1 run, repeat 100 times, take average)
+        times = timeit.repeat(
+            lambda: func(*args, **kwargs),
+            number=1,
+            repeat=100,
+            globals=globals(),
+            timer=time.process_time,
+        )
+        min_time = round(mean(times) * 1000, 2)  # Convert to milliseconds
+        print(f"average execution time: {min_time} ms")
+        return min_time
+
+    return wrapper
+
+
+@benchmark
+def test_lxml():
+    return [
+        e.text
+        for e in etree.fromstring(
+            large_html,
+            # Scrapling and Parsel use the same parser inside, so this is just to make it fair
+            parser=html.HTMLParser(recover=True, huge_tree=True),
+        ).cssselect(".item")
+    ]
+
+
+@benchmark
+def test_bs4_lxml():
+    return [e.text for e in BeautifulSoup(large_html, "lxml").select(".item")]
+
+
+@benchmark
+def test_bs4_html5lib():
+    return [e.text for e in BeautifulSoup(large_html, "html5lib").select(".item")]
+
+
+@benchmark
+def test_pyquery():
+    return [e.text() for e in pq(large_html)(".item").items()]
+
+
+@benchmark
+def test_scrapling():
+    # No need to do `.extract()` like parsel to extract text
+    # Also, this is faster than `[t.text for t in Selector(large_html, adaptive=False).css('.item')]`
+    # for obvious reasons, of course.
+    return ScraplingSelector(large_html, adaptive=False).css(".item::text").getall()
+
+
+@benchmark
+def test_parsel():
+    return Selector(text=large_html).css(".item::text").extract()
+
+
+@benchmark
+def test_mechanicalsoup():
+    browser = StatefulBrowser()
+    browser.open_fake_page(large_html)
+    return [e.text for e in browser.page.select(".item")]
+
+
+@benchmark
+def test_selectolax():
+    return [node.text() for node in HTMLParser(large_html).css(".item")]
+
+
+def display(results):
+    # Sort and display results
+    sorted_results = sorted(results.items(), key=lambda x: x[1])  # Sort by time
+    scrapling_time = results["Scrapling"]
+    print("\nRanked Results (fastest to slowest):")
+    print(f" i. {'Library tested':<18} | {'avg. time (ms)':<15} | vs Scrapling")
+    print("-" * 50)
+    for i, (test_name, test_time) in enumerate(sorted_results, 1):
+        compare = round(test_time / scrapling_time, 3)
+        print(f" {i}. {test_name:<18} | {str(test_time):<15} | {compare}")
+
+
+@benchmark
+def test_scrapling_text(request_html):
+    return ScraplingSelector(request_html, adaptive=False).find_by_text("Tipping the Velvet", first_match=True, clean_match=False).find_similar(ignore_attributes=["title"])
+
+
+@benchmark
+def test_autoscraper(request_html):
+    # autoscraper by default returns elements text
+    return AutoScraper().build(html=request_html, wanted_list=["Tipping the Velvet"])
+
+
+if __name__ == "__main__":
+    print(
+        " Benchmark: Speed of parsing and retrieving the text content of 5000 nested elements \n"
+    )
+    results1 = {
+        "Raw Lxml": test_lxml(),
+        "Parsel/Scrapy": test_parsel(),
+        "Scrapling": test_scrapling(),
+        "Selectolax": test_selectolax(),
+        "PyQuery": test_pyquery(),
+        "BS4 with Lxml": test_bs4_lxml(),
+        "MechanicalSoup": test_mechanicalsoup(),
+        "BS4 with html5lib": test_bs4_html5lib(),
+    }
+
+    display(results1)
+    print("\n" + "=" * 25)
+    req = requests.get("https://books.toscrape.com/index.html")
+    print(
+        " Benchmark: Speed of searching for an element by text content, and retrieving the text of similar elements\n"
+    )
+    results2 = {
+        "Scrapling": test_scrapling_text(req.text),
+        "AutoScraper": test_autoscraper(req.text),
+    }
+    display(results2)

cleanup.py

@@ -0,0 +1,42 @@
+import shutil
+from pathlib import Path
+
+
+# Clean up after installing for local development
+def clean():
+    # Get the current directory
+    base_dir = Path.cwd()
+
+    # Directories and patterns to clean
+    cleanup_patterns = [
+        "build",
+        "dist",
+        "*.egg-info",
+        "__pycache__",
+        ".eggs",
+        ".pytest_cache",
+    ]
+
+    # Clean directories
+    for pattern in cleanup_patterns:
+        for path in base_dir.glob(pattern):
+            try:
+                if path.is_dir():
+                    shutil.rmtree(path)
+                else:
+                    path.unlink()
+                print(f"Removed: {path}")
+            except Exception as e:
+                print(f"Could not remove {path}: {e}")
+
+    # Remove compiled Python files
+    for path in base_dir.rglob("*.py[co]"):
+        try:
+            path.unlink()
+            print(f"Removed compiled file: {path}")
+        except Exception as e:
+            print(f"Could not remove {path}: {e}")
+
+
+if __name__ == "__main__":
+    clean()

docs/README_AR.md

@@ -0,0 +1,426 @@
+<!-- mcp-name: io.github.D4Vinci/Scrapling -->
+
+<h1 align="center">
+    <a href="https://scrapling.readthedocs.io">
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_dark.svg?sanitize=true">
+          <img alt="Scrapling Poster" src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_light.svg?sanitize=true">
+        </picture>
+    </a>
+    <br>
+    <small>Effortless Web Scraping for the Modern Web</small>
+</h1>
+
+<p align="center">
+    <a href="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml" alt="Tests">
+        <img alt="Tests" src="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg"></a>
+    <a href="https://badge.fury.io/py/Scrapling" alt="PyPI version">
+        <img alt="PyPI version" src="https://badge.fury.io/py/Scrapling.svg"></a>
+    <a href="https://pepy.tech/project/scrapling" alt="PyPI Downloads">
+        <img alt="PyPI Downloads" src="https://static.pepy.tech/personalized-badge/scrapling?period=total&units=INTERNATIONAL_SYSTEM&left_color=GREY&right_color=GREEN&left_text=Downloads"></a>
+    <br/>
+    <a href="https://discord.gg/EMgGbDceNQ" alt="Discord" target="_blank">
+      <img alt="Discord" src="https://img.shields.io/discord/1360786381042880532?style=social&logo=discord&link=https%3A%2F%2Fdiscord.gg%2FEMgGbDceNQ">
+    </a>
+    <a href="https://x.com/Scrapling_dev" alt="X (formerly Twitter)">
+      <img alt="X (formerly Twitter) Follow" src="https://img.shields.io/twitter/follow/Scrapling_dev?style=social&logo=x&link=https%3A%2F%2Fx.com%2FScrapling_dev">
+    </a>
+    <br/>
+    <a href="https://pypi.org/project/scrapling/" alt="Supported Python versions">
+        <img alt="Supported Python versions" src="https://img.shields.io/pypi/pyversions/scrapling.svg"></a>
+</p>
+
+<p align="center">
+    <a href="https://scrapling.readthedocs.io/en/latest/parsing/selection/"><strong>طرق الاختيار</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/fetching/choosing/"><strong>اختيار Fetcher</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/architecture.html"><strong>العناكب</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/proxy-blocking.html"><strong>تدوير البروكسي</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/cli/overview/"><strong>واجهة سطر الأوامر</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/ai/mcp-server/"><strong>وضع MCP</strong></a>
+</p>
+
+Scrapling هو إطار عمل تكيفي لـ Web Scraping يتعامل مع كل شيء من طلب واحد إلى زحف كامل النطاق.
+
+محلله يتعلم من تغييرات المواقع ويعيد تحديد موقع عناصرك تلقائياً عند تحديث الصفحات. جوالبه تتجاوز أنظمة مكافحة الروبوتات مثل Cloudflare Turnstile مباشرةً. وإطار عمل Spider الخاص به يتيح لك التوسع إلى عمليات زحف متزامنة ومتعددة الجلسات مع إيقاف/استئناف وتدوير تلقائي لـ Proxy - كل ذلك في بضعة أسطر من Python. مكتبة واحدة، بدون تنازلات.
+
+زحف سريع للغاية مع إحصائيات فورية و Streaming. مبني بواسطة مستخرجي الويب لمستخرجي الويب والمستخدمين العاديين، هناك شيء للجميع.
+
+```python
+from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
+StealthyFetcher.adaptive = True
+p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True)  # احصل على الموقع بشكل خفي!
+products = p.css('.product', auto_save=True)                                        # استخرج بيانات تنجو من تغييرات تصميم الموقع!
+products = p.css('.product', adaptive=True)                                         # لاحقاً، إذا تغيرت بنية الموقع، مرر `adaptive=True` للعثور عليها!
+```
+أو توسع إلى عمليات زحف كاملة
+```python
+from scrapling.spiders import Spider, Response
+
+class MySpider(Spider):
+  name = "demo"
+  start_urls = ["https://example.com/"]
+
+  async def parse(self, response: Response):
+      for item in response.css('.product'):
+          yield {"title": item.css('h2::text').get()}
+
+MySpider().start()
+```
+
+
+# الرعاة البلاتينيون
+
+<i><sub>هل تريد أن تكون أول شركة تظهر هنا؟ انقر [هنا](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
+# الرعاة
+
+<!-- sponsors -->
+
+<a href="https://www.scrapeless.com/en?utm_source=official&utm_term=scrapling" target="_blank" title="Effortless Web Scraping Toolkit for Business and Developers"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/scrapeless.jpg"></a>
+<a href="https://www.thordata.com/?ls=github&lk=github" target="_blank" title="Unblockable proxies and scraping infrastructure, delivering real-time, reliable web data to power AI models and workflows."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/thordata.jpg"></a>
+<a href="https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling" target="_blank" title="Evomi is your Swiss Quality Proxy Provider, starting at $0.49/GB"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/evomi.png"></a>
+<a href="https://serpapi.com/?utm_source=scrapling" target="_blank" title="Scrape Google and other search engines with SerpApi"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SerpApi.png"></a>
+<a href="https://visit.decodo.com/Dy6W0b" target="_blank" title="Try the Most Efficient Residential Proxies for Free"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/decodo.png"></a>
+<a href="https://petrosky.io/d4vinci" target="_blank" title="PetroSky delivers cutting-edge VPS hosting."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/petrosky.png"></a>
+<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
+<a href="https://proxyempire.io/" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
+<a href="https://hypersolutions.co/?utm_source=github&utm_medium=readme&utm_campaign=scrapling" target="_blank" title="Bot Protection Bypass API for Akamai, DataDome, Incapsula & Kasada"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/HyperSolutions.png"></a>
+
+
+<a href="https://www.swiftproxy.net/" target="_blank" title="Unlock Reliable Proxy Services with Swiftproxy!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/swiftproxy.png"></a>
+<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
+<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
+
+<!-- /sponsors -->
+
+<i><sub>هل تريد عرض إعلانك هنا؟ انقر [هنا](https://github.com/sponsors/D4Vinci) واختر المستوى الذي يناسبك!</sub></i>
+
+---
+
+## الميزات الرئيسية
+
+### Spiders — إطار عمل زحف كامل
+- 🕷️ **واجهة Spider شبيهة بـ Scrapy**: عرّف Spiders مع `start_urls`، و async `parse` callbacks، وكائنات `Request`/`Response`.
+- ⚡ **زحف متزامن**: حدود تزامن قابلة للتكوين، وتحكم بالسرعة حسب النطاق، وتأخيرات التنزيل.
+- 🔄 **دعم الجلسات المتعددة**: واجهة موحدة لطلبات HTTP، ومتصفحات خفية بدون واجهة في Spider واحد — وجّه الطلبات إلى جلسات مختلفة بالمعرّف.
+- 💾 **إيقاف واستئناف**: استمرارية الزحف القائمة على Checkpoint. اضغط Ctrl+C للإيقاف بسلاسة؛ أعد التشغيل للاستئناف من حيث توقفت.
+- 📡 **وضع Streaming**: بث العناصر المستخرجة فور وصولها عبر `async for item in spider.stream()` مع إحصائيات فورية — مثالي لواجهات المستخدم وخطوط الأنابيب وعمليات الزحف الطويلة.
+- 🛡️ **كشف الطلبات المحظورة**: كشف تلقائي وإعادة محاولة للطلبات المحظورة مع منطق قابل للتخصيص.
+- 📦 **تصدير مدمج**: صدّر النتائج عبر الخطافات وخط الأنابيب الخاص بك أو JSON/JSONL المدمج مع `result.items.to_json()` / `result.items.to_jsonl()` على التوالي.
+
+### جلب متقدم للمواقع مع دعم الجلسات
+- **طلبات HTTP**: طلبات HTTP سريعة وخفية مع فئة `Fetcher`. يمكنها تقليد بصمة TLS للمتصفح والرؤوس واستخدام HTTP/3.
+- **التحميل الديناميكي**: جلب المواقع الديناميكية مع أتمتة كاملة للمتصفح من خلال فئة `DynamicFetcher` التي تدعم Chromium من Playwright و Google Chrome.
+- **تجاوز مكافحة الروبوتات**: قدرات تخفي متقدمة مع `StealthyFetcher` وا��تحال fingerprint. يمكنه تجاوز جميع أنواع Turnstile/Interstitial من Cloudflare بسهولة بالأتمتة.
+- **إدارة الجلسات**: دعم الجلسات المستمرة مع فئات `FetcherSession` و`StealthySession` و`DynamicSession` لإدارة ملفات تعريف الارتباط والحالة عبر الطلبات.
+- **تدوير Proxy**: `ProxyRotator` مدمج مع استراتيجيات التدوير الدوري أو المخصصة عبر جميع أنواع الجلسات، بالإضافة إلى تجاوزات Proxy لكل طلب.
+- **حظر النطاقات**: حظر الطلبات إلى نطاقات محددة (ونطاقاتها الفرعية) في الجوالب المعتمدة على المتصفح.
+- **دعم Async**: دعم async كامل عبر جميع الجوالب وفئات الجلسات async المخصصة.
+
+### الاستخراج التكيفي والتكامل مع الذكاء الاصطناعي
+- 🔄 **تتبع العناصر الذكي**: إعادة تحديد موقع العناصر بعد تغييرات الموقع باستخدام خوارزميات التشابه الذكية.
+- 🎯 **الاختيار المرن الذكي**: محددات CSS، محددات XPath، البحث القائم على الفلاتر، البحث النصي، البحث بالتعبيرات العادية والمزيد.
+- 🔍 **البحث عن عناصر مشابهة**: تحديد العناصر المشابهة للعناصر الموجودة تلقائياً.
+- 🤖 **خادم MCP للاستخدام مع الذكاء الاصطناعي**: خادم MCP مدمج لـ Web Scraping بمساعدة الذكاء الاصطناعي واستخراج البيانات. يتميز خادم MCP بقدرات قوية مخصصة تستفيد من Scrapling لاستخراج المحتوى المستهدف قبل تمريره إلى الذكاء الاصطناعي (Claude/Cursor/إلخ)، وبالتالي تسريع العمليات وتقليل التكاليف عن طريق تقليل استخدام الرموز. ([فيديو توضيحي](https://www.youtube.com/watch?v=qyFk3ZNwOxE))
+
+### بنية عالية الأداء ومختبرة ميدانياً
+- 🚀 **سريع كالبرق**: أداء محسّن يتفوق على معظم مكتبات Web Scraping في Python.
+- 🔋 **فعال في استخدام الذاكرة**: هياكل بيانات محسّنة وتحميل كسول لأقل استخدام للذاكرة.
+- ⚡ **تسلسل JSON سريع**: أسرع 10 مرات من المكتبة القياسية.
+- 🏗️ **مُختبر ميدانياً**: لا يمتلك Scrapling فقط تغطية اختبار بنسبة 92٪ وتغطية كاملة لتلميحات الأنواع، بل تم استخدامه يومياً من قبل مئات مستخرجي الويب خلال العام الماضي.
+
+### تجربة صديقة للمطورين/مستخرجي الويب
+- 🎯 **Shell تفاعلي لـ Web Scraping**: Shell IPython مدمج اختياري مع تكامل Scrapling، واختصارات، وأدوات جديدة لتسريع تطوير سكريبتات Web Scraping، مثل تحويل طلبات curl إلى طلبات Scrapling وعرض نتائج الطلبات في متصفحك.
+- 🚀 **استخدمه مباشرة من الطرفية**: اختيارياً، يمكنك استخدام Scrapling لاستخراج عنوان URL دون كتابة سطر واحد من الكود!
+- 🛠️ **واجهة تنقل غنية**: اجتياز DOM متقدم مع طرق التنقل بين العناصر الوالدية والشقيقة والفرعية.
+- 🧬 **معالجة نصوص محسّنة**: تعبيرات عادية مدمجة وطرق تنظيف وعمليات نصية محسّنة.
+- 📝 **إنشاء محددات تلقائي**: إنشاء محددات CSS/XPath قوية لأي عنصر.
+- 🔌 **واجهة مألوفة**: مشابه لـ Scrapy/BeautifulSoup مع نفس العناصر الزائفة المستخدمة في Scrapy/Parsel.
+- 📘 **تغطية كاملة للأنواع**: تلميحات نوع كاملة لدعم IDE ممتاز وإكمال الكود. يتم فحص قاعدة الكود بالكامل تلقائياً بواسطة **PyRight** و**MyPy** مع كل تغيير.
+- 🔋 **صورة Docker جاهزة**: مع كل إصدار، يتم بناء ودفع صورة Docker تحتوي على جميع المتصفحات تلقائياً.
+
+## البدء
+
+لنلقِ نظرة سريعة على ما يمكن لـ Scrapling فعله دون التعمق.
+
+### الاستخدام الأساسي
+طلبات HTTP مع دعم الجلسات
+```python
+from scrapling.fetchers import Fetcher, FetcherSession
+
+with FetcherSession(impersonate='chrome') as session:  # استخدم أحدث إصدار من بصمة TLS لـ Chrome
+    page = session.get('https://quotes.toscrape.com/', stealthy_headers=True)
+    quotes = page.css('.quote .text::text').getall()
+
+# أو استخدم طلبات لمرة واحدة
+page = Fetcher.get('https://quotes.toscrape.com/')
+quotes = page.css('.quote .text::text').getall()
+```
+وضع التخفي المتقدم
+```python
+from scrapling.fetchers import StealthyFetcher, StealthySession
+
+with StealthySession(headless=True, solve_cloudflare=True) as session:  # أبقِ المتصفح مفتوحاً حتى تنتهي
+    page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False)
+    data = page.css('#padded_content a').getall()
+
+# أو استخدم نمط الطلب لمرة واحدة، يفتح المتصفح لهذا الطلب، ثم يغلقه بعد الانتهاء
+page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare')
+data = page.css('#padded_content a').getall()
+```
+أتمتة المتصفح الكاملة
+```python
+from scrapling.fetchers import DynamicFetcher, DynamicSession
+
+with DynamicSession(headless=True, disable_resources=False, network_idle=True) as session:  # أبقِ المتصفح مفتوحاً حتى تنتهي
+    page = session.fetch('https://quotes.toscrape.com/', load_dom=False)
+    data = page.xpath('//span[@class="text"]/text()').getall()  # محدد XPath إذا كنت تفضله
+
+# أو استخدم نمط الطلب لمرة واحدة، يفتح المتصفح لهذا الطلب، ثم يغلقه بعد الانتهاء
+page = DynamicFetcher.fetch('https://quotes.toscrape.com/')
+data = page.css('.quote .text::text').getall()
+```
+
+### Spiders
+ابنِ زواحف كاملة مع طلبات متزامنة وأنواع جلسات متعددة وإيقاف/استئناف:
+```python
+from scrapling.spiders import Spider, Request, Response
+
+class QuotesSpider(Spider):
+    name = "quotes"
+    start_urls = ["https://quotes.toscrape.com/"]
+    concurrent_requests = 10
+
+    async def parse(self, response: Response):
+        for quote in response.css('.quote'):
+            yield {
+                "text": quote.css('.text::text').get(),
+                "author": quote.css('.author::text').get(),
+            }
+
+        next_page = response.css('.next a')
+        if next_page:
+            yield response.follow(next_page[0].attrib['href'])
+
+result = QuotesSpider().start()
+print(f"Scraped {len(result.items)} quotes")
+result.items.to_json("quotes.json")
+```
+استخدم أنواع جلسات متعددة في Spider واحد:
+```python
+from scrapling.spiders import Spider, Request, Response
+from scrapling.fetchers import FetcherSession, AsyncStealthySession
+
+class MultiSessionSpider(Spider):
+    name = "multi"
+    start_urls = ["https://example.com/"]
+
+    def configure_sessions(self, manager):
+        manager.add("fast", FetcherSession(impersonate="chrome"))
+        manager.add("stealth", AsyncStealthySession(headless=True), lazy=True)
+
+    async def parse(self, response: Response):
+        for link in response.css('a::attr(href)').getall():
+            # وجّه الصفحات المحمية عبر جلسة التخفي
+            if "protected" in link:
+                yield Request(link, sid="stealth")
+            else:
+                yield Request(link, sid="fast", callback=self.parse)  # callback صريح
+```
+أوقف واستأنف عمليات الزحف الطويلة مع Checkpoints بتشغيل Spider هكذا:
+```python
+QuotesSpider(crawldir="./crawl_data").start()
+```
+اضغط Ctrl+C للإيقاف بسلاسة — يتم حفظ التقدم تلقائياً. لاحقاً، عند تشغيل Spider مرة أخرى، مرر نفس `crawldir`، وسيستأنف من حيث توقف.
+
+### التحليل المتقدم والتنقل
+```python
+from scrapling.fetchers import Fetcher
+
+# اختيار عناصر غني وتنقل
+page = Fetcher.get('https://quotes.toscrape.com/')
+
+# احصل على الاقتباسات بطرق اختيار متعددة
+quotes = page.css('.quote')  # محدد CSS
+quotes = page.xpath('//div[@class="quote"]')  # XPath
+quotes = page.find_all('div', {'class': 'quote'})  # بأسلوب BeautifulSoup
+# نفس الشيء مثل
+quotes = page.find_all('div', class_='quote')
+quotes = page.find_all(['div'], class_='quote')
+quotes = page.find_all(class_='quote')  # وهكذا...
+# البحث عن عنصر بمحتوى النص
+quotes = page.find_by_text('quote', tag='div')
+
+# التنقل المتقدم
+quote_text = page.css('.quote')[0].css('.text::text').get()
+quote_text = page.css('.quote').css('.text::text').getall()  # محددات متسلسلة
+first_quote = page.css('.quote')[0]
+author = first_quote.next_sibling.css('.author::text')
+parent_container = first_quote.parent
+
+# علاقات العناصر والتشابه
+similar_elements = first_quote.find_similar()
+below_elements = first_quote.below_elements()
+```
+يمكنك استخدام المحلل مباشرة إذا كنت لا تريد جلب المواقع كما يلي:
+```python
+from scrapling.parser import Selector
+
+page = Selector("<html>...</html>")
+```
+وهو يعمل بنفس الطريقة تماماً!
+
+### أمثلة إدارة ��لجلسات بشكل Async
+```python
+import asyncio
+from scrapling.fetchers import FetcherSession, AsyncStealthySession, AsyncDynamicSession
+
+async with FetcherSession(http3=True) as session:  # `FetcherSession` واعٍ بالسياق ويعمل في كلا النمطين المتزامن/async
+    page1 = session.get('https://quotes.toscrape.com/')
+    page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135')
+
+# استخدام جلسة async
+async with AsyncStealthySession(max_pages=2) as session:
+    tasks = []
+    urls = ['https://example.com/page1', 'https://example.com/page2']
+
+    for url in urls:
+        task = session.fetch(url)
+        tasks.append(task)
+
+    print(session.get_pool_stats())  # اختياري - حالة مجموعة علامات تبويب المتصفح (مشغول/حر/خطأ)
+    results = await asyncio.gather(*tasks)
+    print(session.get_pool_stats())
+```
+
+## واجهة سطر الأوامر والـ Shell التفاعلي
+
+يتضمن Scrapling واجهة سطر أوامر قوية:
+
+[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339)
+
+تشغيل Shell الـ Web Scraping التفاعلي
+```bash
+scrapling shell
+```
+استخرج الصفحات إلى ملف مباشرة دون برمجة (يستخرج المحتوى داخل وسم `body` افتراضياً). إذا انتهى ملف الإخراج بـ `.txt`، فسيتم استخراج محتوى النص للهدف. إذا انتهى بـ `.md`، فسيكون تمثيل Markdown لمحتوى HTML؛ إذا انتهى بـ `.html`، فسيكون محتوى HTML نفسه.
+```bash
+scrapling extract get 'https://example.com' content.md
+scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome'  # جميع العناصر المطابقة لمحدد CSS '#fromSkipToProducts'
+scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless
+scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare
+```
+
+> [!NOTE]
+> هناك العديد من الميزات الإضافية، لكننا نريد إبقاء هذه الصفحة موجزة، بما في ذلك خادم MCP والـ Shell التفاعلي لـ Web Scraping. تحقق من الوثائق الكاملة [هنا](https://scrapling.readthedocs.io/en/latest/)
+
+## معايير الأداء
+
+Scrapling ليس قوياً فحسب — بل هو أيضاً سريع بشكل مذهل. تقارن المعايير التالية محلل Scrapling مع أحدث إصدارات المكتبات الشائعة الأخرى.
+
+### اختبار سرعة استخراج النص (5000 عنصر متداخل)
+
+| # |      المكتبة      | الوقت (ms) | vs Scrapling |
+|---|:-----------------:|:----------:|:------------:|
+| 1 |     Scrapling     |    2.02    |     1.0x     |
+| 2 |   Parsel/Scrapy   |    2.04    |     1.01     |
+| 3 |     Raw Lxml      |    2.54    |    1.257     |
+| 4 |      PyQuery      |   24.17    |     ~12x     |
+| 5 |    Selectolax     |   82.63    |     ~41x     |
+| 6 |  MechanicalSoup   |  1549.71   |   ~767.1x    |
+| 7 |   BS4 with Lxml   |  1584.31   |   ~784.3x    |
+| 8 | BS4 with html5lib |  3391.91   |   ~1679.1x   |
+
+
+### أداء تشابه العناصر والبحث النصي
+
+قدرات العثور على العناصر التكيفية لـ Scrapling تتفوق بشكل كبير على البدائل:
+
+| المكتبة     | الوقت (ms) | vs Scrapling |
+|-------------|:----------:|:------------:|
+| Scrapling   |    2.39    |     1.0x     |
+| AutoScraper |   12.45    |    5.209x    |
+
+
+> تمثل جميع المعايير متوسطات أكثر من 100 تشغيل. انظر [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) للمنهجية.
+
+## التثبيت
+
+يتطلب Scrapling إصدار Python 3.10 أو أعلى:
+
+```bash
+pip install scrapling
+```
+
+يتضمن هذا التثبيت فقط محرك المحلل وتبعياته، بدون أي جوالب أو تبعيات سطر الأوامر.
+
+### التبعيات الاختيارية
+
+1. إذا كنت ستستخدم أياً من الميزات الإضافية أدناه، أو الجوالب، أو فئاتها، فستحتاج إلى تثبيت تبعيات الجوالب وتبعيات المتصفح الخاصة بها على النحو التالي:
+    ```bash
+    pip install "scrapling[fetchers]"
+
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
+    ```
+
+    يقوم هذا بتنزيل جميع المتصفحات، إلى جانب تبعيات النظام وتبعيات معالجة fingerprint الخاصة بها.
+
+    أو يمكنك تثبيتها من الكود بدلاً من تشغيل أمر كالتالي:
+    ```python
+    from scrapling.cli import install
+
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
+2. ميزات إضافية:
+   - تثبيت ميزة خادم MCP:
+       ```bash
+       pip install "scrapling[ai]"
+       ```
+   - تثبيت ميزات Shell (Shell الـ Web Scraping وأمر `extract`):
+       ```bash
+       pip install "scrapling[shell]"
+       ```
+   - تثبيت كل شيء:
+       ```bash
+       pip install "scrapling[all]"
+       ```
+   تذكر أنك تحتاج إلى تثبيت تبعيات المتصفح مع `scrapling install` بعد أي من هذه الإضافات (إذا لم تكن قد فعلت ذلك بالفعل)
+
+### Docker
+يمكنك أيضاً تثبيت صورة Docker مع جميع الإضافات والمتصفحات باستخدام الأمر التالي من DockerHub:
+```bash
+docker pull pyd4vinci/scrapling
+```
+أو تنزيلها من سجل GitHub:
+```bash
+docker pull ghcr.io/d4vinci/scrapling:latest
+```
+يتم بناء هذه الصورة ودفعها تلقائياً باستخدام GitHub Actions والفرع الرئيسي للمستودع.
+
+## المساهمة
+
+نرحب بالمساهمات! يرجى قراءة [إرشادات المساهمة](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md) قبل البدء.
+
+## إخلاء المسؤولية
+
+> [!CAUTION]
+> يتم توفير هذه المكتبة للأغراض التعليمية والبحثية فقط. باستخدام هذه المكتبة، فإنك توافق على الامتثال لقوانين استخراج البيانات والخصوصية المحلية والدولية. المؤلفون والمساهمون غير مسؤولين عن أي إساءة استخدام لهذا البرنامج. احترم دائماً شروط خدمة المواقع وملفات robots.txt.
+
+## الترخيص
+
+هذا العمل مرخص بموجب ترخيص BSD-3-Clause.
+
+## الشكر والتقدير
+
+يتضمن هذا المشروع كوداً معدلاً من:
+- Parsel (ترخيص BSD) — يُستخدم للوحدة الفرعية [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py)
+
+---
+<div align="center"><small>مصمم ومصنوع بـ ❤️ بواسطة كريم شعير.</small></div><br>

docs/README_CN.md

@@ -0,0 +1,426 @@
+<!-- mcp-name: io.github.D4Vinci/Scrapling -->
+
+<h1 align="center">
+    <a href="https://scrapling.readthedocs.io">
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_dark.svg?sanitize=true">
+          <img alt="Scrapling Poster" src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_light.svg?sanitize=true">
+        </picture>
+    </a>
+    <br>
+    <small>Effortless Web Scraping for the Modern Web</small>
+</h1>
+
+<p align="center">
+    <a href="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml" alt="Tests">
+        <img alt="Tests" src="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg"></a>
+    <a href="https://badge.fury.io/py/Scrapling" alt="PyPI version">
+        <img alt="PyPI version" src="https://badge.fury.io/py/Scrapling.svg"></a>
+    <a href="https://pepy.tech/project/scrapling" alt="PyPI Downloads">
+        <img alt="PyPI Downloads" src="https://static.pepy.tech/personalized-badge/scrapling?period=total&units=INTERNATIONAL_SYSTEM&left_color=GREY&right_color=GREEN&left_text=Downloads"></a>
+    <br/>
+    <a href="https://discord.gg/EMgGbDceNQ" alt="Discord" target="_blank">
+      <img alt="Discord" src="https://img.shields.io/discord/1360786381042880532?style=social&logo=discord&link=https%3A%2F%2Fdiscord.gg%2FEMgGbDceNQ">
+    </a>
+    <a href="https://x.com/Scrapling_dev" alt="X (formerly Twitter)">
+      <img alt="X (formerly Twitter) Follow" src="https://img.shields.io/twitter/follow/Scrapling_dev?style=social&logo=x&link=https%3A%2F%2Fx.com%2FScrapling_dev">
+    </a>
+    <br/>
+    <a href="https://pypi.org/project/scrapling/" alt="Supported Python versions">
+        <img alt="Supported Python versions" src="https://img.shields.io/pypi/pyversions/scrapling.svg"></a>
+</p>
+
+<p align="center">
+    <a href="https://scrapling.readthedocs.io/en/latest/parsing/selection/"><strong>选择方法</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/fetching/choosing/"><strong>选择Fetcher</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/architecture.html"><strong>爬虫</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/proxy-blocking.html"><strong>代理轮换</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/cli/overview/"><strong>CLI</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/ai/mcp-server/"><strong>MCP模式</strong></a>
+</p>
+
+Scrapling是一个自适应Web Scraping框架，能处理从单个请求到大规模爬取的一切需求。
+
+它的解析器能够从网站变化中学习，并在页面更新时自动重新定位您的元素。它的Fetcher能够开箱即用地绕过Cloudflare Turnstile等反机器人系统。它的Spider框架让您可以扩展到并发、多Session爬取，支持暂停/恢复和自动Proxy轮换——只需几行Python代码。一个库，零妥协。
+
+极速爬取，实时统计和Streaming。由Web Scraper为Web Scraper和普通用户而构建，每个人都能找到适合自己的功能。
+
+```python
+from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
+StealthyFetcher.adaptive = True
+p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True)  # 隐秘地获取网站！
+products = p.css('.product', auto_save=True)                                        # 抓取在网站设计变更后仍能存活的数据！
+products = p.css('.product', adaptive=True)                                         # 之后，如果网站结构改变，传递 `adaptive=True` 来找到它们！
+```
+或扩展为完整爬取
+```python
+from scrapling.spiders import Spider, Response
+
+class MySpider(Spider):
+  name = "demo"
+  start_urls = ["https://example.com/"]
+
+  async def parse(self, response: Response):
+      for item in response.css('.product'):
+          yield {"title": item.css('h2::text').get()}
+
+MySpider().start()
+```
+
+
+# 铂金赞助商
+
+<i><sub>想成为第一个出现在这里的公司吗？点击[这里](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
+# 赞助商
+
+<!-- sponsors -->
+
+<a href="https://www.scrapeless.com/en?utm_source=official&utm_term=scrapling" target="_blank" title="Effortless Web Scraping Toolkit for Business and Developers"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/scrapeless.jpg"></a>
+<a href="https://www.thordata.com/?ls=github&lk=github" target="_blank" title="Unblockable proxies and scraping infrastructure, delivering real-time, reliable web data to power AI models and workflows."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/thordata.jpg"></a>
+<a href="https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling" target="_blank" title="Evomi is your Swiss Quality Proxy Provider, starting at $0.49/GB"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/evomi.png"></a>
+<a href="https://serpapi.com/?utm_source=scrapling" target="_blank" title="Scrape Google and other search engines with SerpApi"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SerpApi.png"></a>
+<a href="https://visit.decodo.com/Dy6W0b" target="_blank" title="Try the Most Efficient Residential Proxies for Free"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/decodo.png"></a>
+<a href="https://petrosky.io/d4vinci" target="_blank" title="PetroSky delivers cutting-edge VPS hosting."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/petrosky.png"></a>
+<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
+<a href="https://proxyempire.io/" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
+<a href="https://hypersolutions.co/?utm_source=github&utm_medium=readme&utm_campaign=scrapling" target="_blank" title="Bot Protection Bypass API for Akamai, DataDome, Incapsula & Kasada"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/HyperSolutions.png"></a>
+
+
+<a href="https://www.swiftproxy.net/" target="_blank" title="Unlock Reliable Proxy Services with Swiftproxy!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/swiftproxy.png"></a>
+<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
+<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
+
+<!-- /sponsors -->
+
+<i><sub>想在这里展示您的广告吗？点击[这里](https://github.com/sponsors/D4Vinci)并选择适合您的级别！</sub></i>
+
+---
+
+## 主要特性
+
+### Spider — 完整的爬取框架
+- 🕷️ **类Scrapy的Spider API**：使用`start_urls`、async `parse` callback和`Request`/`Response`对象定义Spider。
+- ⚡ **并发爬取**：可配置的并发限制、按域名节流和下载延迟。
+- 🔄 **多Session支持**：统一接口，支持HTTP请求和隐秘无头浏览器在同一个Spider中使用——通过ID将请求路由到不同的Session。
+- 💾 **暂停与恢复**：基于Checkpoint的爬取持久化。按Ctrl+C优雅关闭；重启后从上次停止的地方继续。
+- 📡 **Streaming模式**：通过`async for item in spider.stream()`以实时统计Streaming抓取的数据——非常适合UI、管道和长时间运行的爬取。
+- 🛡️ **被阻止请求检测**：自动检测并重试被阻止的请求，支持自定义逻辑。
+- 📦 **内置导出**：通过钩子和您自己的管道导出结果，或使用内置的JSON/JSONL，分别通过`result.items.to_json()`/`result.items.to_jsonl()`。
+
+### 支持Session的高级网站获取
+- **HTTP请求**：使用`Fetcher`类进行快速和隐秘的HTTP请求。可以模拟浏览器的TLS fingerprint、标头并使用HTTP/3。
+- **动态加载**：通过`DynamicFetcher`类使用完整的浏览器自动化获取动态网站，支持Playwright的Chromium和Google Chrome。
+- **反机器人绕过**：使用`StealthyFetcher`的高级隐秘功能和fingerprint伪装。可以轻松自动绕过所有类型的Cloudflare Turnstile/Interstitial。
+- **Session管理**：使用`FetcherSession`、`StealthySession`和`DynamicSession`类实现持久化Session支持，用于跨请求的cookie和状态管理。
+- **Proxy轮换**：内置`ProxyRotator`，支持轮询或自定义策略，适用于所有Session类型，并支持按请求覆盖Proxy。
+- **域名屏蔽**：在基于浏览器的Fetcher中屏蔽对特定域名（及其子域名）的请求。
+- **Async支持**：所有Fetcher和专用async Session类的完整async支持。
+
+### 自适应抓取和AI集成
+- 🔄 **智能元素跟踪**：使用智能相似性算法在网站更改后重新定位元素。
+- 🎯 **智能灵活选择**：CSS选择器、XPath选择器、基于过滤器的搜索、文本搜索、正则表达式搜索等。
+- 🔍 **查找相似元素**：自动定位与已找到元素相似的元素。
+- 🤖 **与AI一起使用的MCP服务器**：内置MCP服务器用于AI辅助Web Scraping和数据提取。MCP服务器具有强大的自定义功能，利用Scrapling在将内容传递给AI（Claude/Cursor等）之前提取目标内容，从而加快操作并通过最小化token使用来降低成本。（[演示视频](https://www.youtube.com/watch?v=qyFk3ZNwOxE)）
+
+### 高性能和经过实战测试的架构
+- 🚀 **闪电般快速**：优化性能超越大多数Python抓取库。
+- 🔋 **内存高效**：优化的数据结构和延迟加载，最小内存占用。
+- ⚡ **快速JSON序列化**：比标准库快10倍。
+- 🏗️ **经过实战测试**：Scrapling不仅拥有92%的测试覆盖率和完整的类型提示覆盖率，而且在过去一年中每天被数百名Web Scraper使用。
+
+### 对开发者/Web Scraper友好的体验
+- 🎯 **交互式Web Scraping Shell**：可选的内置IPython Shell，具有Scrapling集成、快捷方式和新工具，可加快Web Scraping脚本开发，例如将curl请求转换为Scrapling请求并在浏览器中查看请求结果。
+- 🚀 **直接从终端使用**：可选地，您可以使用Scrapling抓取URL而无需编写任何代码！
+- 🛠️ **丰富的导航API**：使用父级、兄弟级和子级导航方法进行高级DOM遍历。
+- 🧬 **增强的文本处理**：内置正则表达式、清理方法和优化的字符串操作。
+- 📝 **自动选择器生成**：为任何元素生成强大的CSS/XPath选择器。
+- 🔌 **熟悉的API**：类似于Scrapy/BeautifulSoup，使用与Scrapy/Parsel相同的伪元素。
+- 📘 **完整的类型覆盖**：完整的类型提示，出色的IDE支持和代码补全。整个代码库在每次更改时都会自动使用**PyRight**和**MyPy**扫描。
+- 🔋 **现成的Docker镜像**：每次发布时，包含所有浏览器的Docker镜像会自动构建和推送。
+
+## 入门
+
+让我们快速展示Scrapling的功能，无需深入了解。
+
+### 基本用法
+支持Session的HTTP请求
+```python
+from scrapling.fetchers import Fetcher, FetcherSession
+
+with FetcherSession(impersonate='chrome') as session:  # 使用Chrome的最新版本TLS fingerprint
+    page = session.get('https://quotes.toscrape.com/', stealthy_headers=True)
+    quotes = page.css('.quote .text::text').getall()
+
+# 或使用一次性请求
+page = Fetcher.get('https://quotes.toscrape.com/')
+quotes = page.css('.quote .text::text').getall()
+```
+高级隐秘模式
+```python
+from scrapling.fetchers import StealthyFetcher, StealthySession
+
+with StealthySession(headless=True, solve_cloudflare=True) as session:  # 保持浏览器打开直到完成
+    page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False)
+    data = page.css('#padded_content a').getall()
+
+# 或使用一次性请求样式，为此请求打开浏览器，完成后关闭
+page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare')
+data = page.css('#padded_content a').getall()
+```
+完整的浏览器自动化
+```python
+from scrapling.fetchers import DynamicFetcher, DynamicSession
+
+with DynamicSession(headless=True, disable_resources=False, network_idle=True) as session:  # 保持浏览器打开直到完成
+    page = session.fetch('https://quotes.toscrape.com/', load_dom=False)
+    data = page.xpath('//span[@class="text"]/text()').getall()  # 如果您偏好XPath选择器
+
+# 或使用一次性请求样式，为此请求打开浏览器，完成后关闭
+page = DynamicFetcher.fetch('https://quotes.toscrape.com/')
+data = page.css('.quote .text::text').getall()
+```
+
+### Spider
+构建具有并发请求、多种Session类型和暂停/恢复功能的完整爬虫：
+```python
+from scrapling.spiders import Spider, Request, Response
+
+class QuotesSpider(Spider):
+    name = "quotes"
+    start_urls = ["https://quotes.toscrape.com/"]
+    concurrent_requests = 10
+
+    async def parse(self, response: Response):
+        for quote in response.css('.quote'):
+            yield {
+                "text": quote.css('.text::text').get(),
+                "author": quote.css('.author::text').get(),
+            }
+
+        next_page = response.css('.next a')
+        if next_page:
+            yield response.follow(next_page[0].attrib['href'])
+
+result = QuotesSpider().start()
+print(f"抓取了 {len(result.items)} 条引用")
+result.items.to_json("quotes.json")
+```
+在单个Spider中使用多种Session类型：
+```python
+from scrapling.spiders import Spider, Request, Response
+from scrapling.fetchers import FetcherSession, AsyncStealthySession
+
+class MultiSessionSpider(Spider):
+    name = "multi"
+    start_urls = ["https://example.com/"]
+
+    def configure_sessions(self, manager):
+        manager.add("fast", FetcherSession(impersonate="chrome"))
+        manager.add("stealth", AsyncStealthySession(headless=True), lazy=True)
+
+    async def parse(self, response: Response):
+        for link in response.css('a::attr(href)').getall():
+            # 将受保护的页面路由到隐秘Session
+            if "protected" in link:
+                yield Request(link, sid="stealth")
+            else:
+                yield Request(link, sid="fast", callback=self.parse)  # 显式callback
+```
+通过如下方式运行Spider来暂停和恢复长时间爬取，使用Checkpoint：
+```python
+QuotesSpider(crawldir="./crawl_data").start()
+```
+按Ctrl+C优雅暂停——进度会自动保存。之后，当您再次启动Spider时，传递相同的`crawldir`，它将从上次停止的地方继续。
+
+### 高级解析与导航
+```python
+from scrapling.fetchers import Fetcher
+
+# 丰富的元素选择和导航
+page = Fetcher.get('https://quotes.toscrape.com/')
+
+# 使用多种选择方法获取引用
+quotes = page.css('.quote')  # CSS选择器
+quotes = page.xpath('//div[@class="quote"]')  # XPath
+quotes = page.find_all('div', {'class': 'quote'})  # BeautifulSoup风格
+# 等同于
+quotes = page.find_all('div', class_='quote')
+quotes = page.find_all(['div'], class_='quote')
+quotes = page.find_all(class_='quote')  # 等等...
+# 按文本内容查找元素
+quotes = page.find_by_text('quote', tag='div')
+
+# 高级导航
+quote_text = page.css('.quote')[0].css('.text::text').get()
+quote_text = page.css('.quote').css('.text::text').getall()  # 链式选择器
+first_quote = page.css('.quote')[0]
+author = first_quote.next_sibling.css('.author::text')
+parent_container = first_quote.parent
+
+# 元素关系和相似性
+similar_elements = first_quote.find_similar()
+below_elements = first_quote.below_elements()
+```
+如果您不想获取网站，可以直接使用解析器，如下所示：
+```python
+from scrapling.parser import Selector
+
+page = Selector("<html>...</html>")
+```
+用法完全相同！
+
+### Async Session管理示例
+```python
+import asyncio
+from scrapling.fetchers import FetcherSession, AsyncStealthySession, AsyncDynamicSession
+
+async with FetcherSession(http3=True) as session:  # `FetcherSession`是上下文感知的，可以在sync/async模式下工作
+    page1 = session.get('https://quotes.toscrape.com/')
+    page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135')
+
+# Async Session用法
+async with AsyncStealthySession(max_pages=2) as session:
+    tasks = []
+    urls = ['https://example.com/page1', 'https://example.com/page2']
+
+    for url in urls:
+        task = session.fetch(url)
+        tasks.append(task)
+
+    print(session.get_pool_stats())  # 可选 - 浏览器标签池的状态（忙/空闲/错误）
+    results = await asyncio.gather(*tasks)
+    print(session.get_pool_stats())
+```
+
+## CLI和交互式Shell
+
+Scrapling包含强大的命令行界面：
+
+[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339)
+
+启动交互式Web Scraping Shell
+```bash
+scrapling shell
+```
+直接将页面提取到文件而无需编程（默认提取`body`标签内的内容）。如果输出文件以`.txt`结尾，则将提取目标的文本内容。如果以`.md`结尾，它将是HTML内容的Markdown表示；如果以`.html`结尾，它将是HTML内容本身。
+```bash
+scrapling extract get 'https://example.com' content.md
+scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome'  # 所有匹配CSS选择器'#fromSkipToProducts'的元素
+scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless
+scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare
+```
+
+> [!NOTE]
+> 还有许多其他功能，但我们希望保持此页面简洁，包括MCP服务器和交互式Web Scraping Shell。查看完整文档[这里](https://scrapling.readthedocs.io/en/latest/)
+
+## 性能基准
+
+Scrapling不仅功能强大——它还速度极快。以下基准测试将Scrapling的解析器与其他流行库的最新版本进行了比较。
+
+### 文本提取速度测试（5000个嵌套元素）
+
+| # |         库         | 时间(ms)  | vs Scrapling |
+|---|:-----------------:|:---------:|:------------:|
+| 1 |     Scrapling     |   2.02    |     1.0x     |
+| 2 |   Parsel/Scrapy   |   2.04    |     1.01     |
+| 3 |     Raw Lxml      |   2.54    |    1.257     |
+| 4 |      PyQuery      |   24.17   |     ~12x     |
+| 5 |    Selectolax     |   82.63   |     ~41x     |
+| 6 |  MechanicalSoup   |  1549.71  |   ~767.1x    |
+| 7 |   BS4 with Lxml   |  1584.31  |   ~784.3x    |
+| 8 | BS4 with html5lib |  3391.91  |   ~1679.1x   |
+
+
+### 元素相似性和文本搜索性能
+
+Scrapling的自适应元素查找功能明显优于替代方案：
+
+| 库           | 时间(ms) | vs Scrapling |
+|-------------|:---------:|:------------:|
+| Scrapling   |   2.39    |     1.0x     |
+| AutoScraper |   12.45   |    5.209x    |
+
+
+> 所有基准测试代表100+次运行的平均值。请参阅[benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py)了解方法。
+
+## 安装
+
+Scrapling需要Python 3.10或更高版本：
+
+```bash
+pip install scrapling
+```
+
+此安装仅包括解析器引擎及其依赖项，没有任何Fetcher或命令行依赖项。
+
+### 可选依赖项
+
+1. 如果您要使用以下任何额外功能、Fetcher或它们的类，您将需要安装Fetcher的依赖项和它们的浏览器依赖项，如下所示：
+    ```bash
+    pip install "scrapling[fetchers]"
+
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
+    ```
+
+    这会下载所有浏览器，以及它们的系统依赖项和fingerprint操作依赖项。
+
+    或者你可以从代码中安装，而不是运行命令：
+    ```python
+    from scrapling.cli import install
+
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
+2. 额外功能：
+   - 安装MCP服务器功能：
+       ```bash
+       pip install "scrapling[ai]"
+       ```
+   - 安装Shell功能（Web Scraping Shell和`extract`命令）：
+       ```bash
+       pip install "scrapling[shell]"
+       ```
+   - 安装所有内容：
+       ```bash
+       pip install "scrapling[all]"
+       ```
+   请记住，在安装任何这些额外功能后（如果您还没有安装），您需要使用`scrapling install`安装浏览器依赖项
+
+### Docker
+您还可以使用以下命令从DockerHub安装包含所有额外功能和浏览器的Docker镜像：
+```bash
+docker pull pyd4vinci/scrapling
+```
+或从GitHub注册表下载：
+```bash
+docker pull ghcr.io/d4vinci/scrapling:latest
+```
+此镜像使用GitHub Actions和仓库主分支自动构建和推送。
+
+## 贡献
+
+我们欢迎贡献！在开始之前，请阅读我们的[贡献指南](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md)。
+
+## 免责声明
+
+> [!CAUTION]
+> 此库仅用于教育和研究目的。使用此库即表示您同意遵守本地和国际数据抓取和隐私法律。作者和贡献者对本软件的任何滥用不承担责任。始终尊重网站的服务条款和robots.txt文件。
+
+## 许可证
+
+本作品根据BSD-3-Clause许可证授权。
+
+## 致谢
+
+此项目包含改编自以下内容的代码：
+- Parsel（BSD许可证）——用于[translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py)子模块
+
+---
+<div align="center"><small>由Karim Shoair用❤️设计和制作。</small></div><br>

docs/README_DE.md

@@ -0,0 +1,426 @@
+<!-- mcp-name: io.github.D4Vinci/Scrapling -->
+
+<h1 align="center">
+    <a href="https://scrapling.readthedocs.io">
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_dark.svg?sanitize=true">
+          <img alt="Scrapling Poster" src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_light.svg?sanitize=true">
+        </picture>
+    </a>
+    <br>
+    <small>Effortless Web Scraping for the Modern Web</small>
+</h1>
+
+<p align="center">
+    <a href="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml" alt="Tests">
+        <img alt="Tests" src="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg"></a>
+    <a href="https://badge.fury.io/py/Scrapling" alt="PyPI version">
+        <img alt="PyPI version" src="https://badge.fury.io/py/Scrapling.svg"></a>
+    <a href="https://pepy.tech/project/scrapling" alt="PyPI Downloads">
+        <img alt="PyPI Downloads" src="https://static.pepy.tech/personalized-badge/scrapling?period=total&units=INTERNATIONAL_SYSTEM&left_color=GREY&right_color=GREEN&left_text=Downloads"></a>
+    <br/>
+    <a href="https://discord.gg/EMgGbDceNQ" alt="Discord" target="_blank">
+      <img alt="Discord" src="https://img.shields.io/discord/1360786381042880532?style=social&logo=discord&link=https%3A%2F%2Fdiscord.gg%2FEMgGbDceNQ">
+    </a>
+    <a href="https://x.com/Scrapling_dev" alt="X (formerly Twitter)">
+      <img alt="X (formerly Twitter) Follow" src="https://img.shields.io/twitter/follow/Scrapling_dev?style=social&logo=x&link=https%3A%2F%2Fx.com%2FScrapling_dev">
+    </a>
+    <br/>
+    <a href="https://pypi.org/project/scrapling/" alt="Supported Python versions">
+        <img alt="Supported Python versions" src="https://img.shields.io/pypi/pyversions/scrapling.svg"></a>
+</p>
+
+<p align="center">
+    <a href="https://scrapling.readthedocs.io/en/latest/parsing/selection/"><strong>Auswahlmethoden</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/fetching/choosing/"><strong>Einen Fetcher wählen</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/architecture.html"><strong>Spiders</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/proxy-blocking.html"><strong>Proxy-Rotation</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/cli/overview/"><strong>CLI</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/ai/mcp-server/"><strong>MCP-Modus</strong></a>
+</p>
+
+Scrapling ist ein adaptives Web-Scraping-Framework, das alles abdeckt -- von einer einzelnen Anfrage bis hin zu einem umfassenden Crawl.
+
+Sein Parser lernt aus Website-Änderungen und lokalisiert Ihre Elemente automatisch neu, wenn sich Seiten aktualisieren. Seine Fetcher umgehen Anti-Bot-Systeme wie Cloudflare Turnstile direkt ab Werk. Und sein Spider-Framework ermöglicht es Ihnen, auf parallele Multi-Session-Crawls mit Pause & Resume und automatischer Proxy-Rotation hochzuskalieren -- alles in wenigen Zeilen Python. Eine Bibliothek, keine Kompromisse.
+
+Blitzschnelle Crawls mit Echtzeit-Statistiken und Streaming. Von Web Scrapern für Web Scraper und normale Benutzer entwickelt, ist für jeden etwas dabei.
+
+```python
+from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
+StealthyFetcher.adaptive = True
+p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True)  # Website unbemerkt abrufen!
+products = p.css('.product', auto_save=True)                                        # Daten scrapen, die Website-Designänderungen überleben!
+products = p.css('.product', adaptive=True)                                         # Später, wenn sich die Website-Struktur ändert, `adaptive=True` übergeben, um sie zu finden!
+```
+Oder auf vollständige Crawls hochskalieren
+```python
+from scrapling.spiders import Spider, Response
+
+class MySpider(Spider):
+  name = "demo"
+  start_urls = ["https://example.com/"]
+
+  async def parse(self, response: Response):
+      for item in response.css('.product'):
+          yield {"title": item.css('h2::text').get()}
+
+MySpider().start()
+```
+
+
+# Platin-Sponsoren
+
+<i><sub>Möchten Sie das erste Unternehmen sein, das hier erscheint? Klicken Sie [hier](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
+# Sponsoren
+
+<!-- sponsors -->
+
+<a href="https://www.scrapeless.com/en?utm_source=official&utm_term=scrapling" target="_blank" title="Effortless Web Scraping Toolkit for Business and Developers"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/scrapeless.jpg"></a>
+<a href="https://www.thordata.com/?ls=github&lk=github" target="_blank" title="Unblockable proxies and scraping infrastructure, delivering real-time, reliable web data to power AI models and workflows."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/thordata.jpg"></a>
+<a href="https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling" target="_blank" title="Evomi is your Swiss Quality Proxy Provider, starting at $0.49/GB"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/evomi.png"></a>
+<a href="https://serpapi.com/?utm_source=scrapling" target="_blank" title="Scrape Google and other search engines with SerpApi"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SerpApi.png"></a>
+<a href="https://visit.decodo.com/Dy6W0b" target="_blank" title="Try the Most Efficient Residential Proxies for Free"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/decodo.png"></a>
+<a href="https://petrosky.io/d4vinci" target="_blank" title="PetroSky delivers cutting-edge VPS hosting."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/petrosky.png"></a>
+<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
+<a href="https://proxyempire.io/" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
+<a href="https://hypersolutions.co/?utm_source=github&utm_medium=readme&utm_campaign=scrapling" target="_blank" title="Bot Protection Bypass API for Akamai, DataDome, Incapsula & Kasada"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/HyperSolutions.png"></a>
+
+
+<a href="https://www.swiftproxy.net/" target="_blank" title="Unlock Reliable Proxy Services with Swiftproxy!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/swiftproxy.png"></a>
+<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
+<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
+
+<!-- /sponsors -->
+
+<i><sub>Möchten Sie Ihre Anzeige hier zeigen? Klicken Sie [hier](https://github.com/sponsors/D4Vinci) und wählen Sie die Stufe, die zu Ihnen passt!</sub></i>
+
+---
+
+## Hauptmerkmale
+
+### Spiders -- Ein vollständiges Crawling-Framework
+- 🕷️ **Scrapy-ähnliche Spider-API**: Definieren Sie Spiders mit `start_urls`, async `parse` Callbacks und `Request`/`Response`-Objekten.
+- ⚡ **Paralleles Crawling**: Konfigurierbare Parallelitätslimits, domainbezogenes Throttling und Download-Verzögerungen.
+- 🔄 **Multi-Session-Unterstützung**: Einheitliche Schnittstelle für HTTP-Anfragen und heimliche Headless-Browser in einem einzigen Spider -- leiten Sie Anfragen per ID an verschiedene Sessions weiter.
+- 💾 **Pause & Resume**: Checkpoint-basierte Crawl-Persistenz. Drücken Sie Strg+C für ein kontrolliertes Herunterfahren; starten Sie neu, um dort fortzufahren, wo Sie aufgehört haben.
+- 📡 **Streaming-Modus**: Gescrapte Elemente in Echtzeit streamen über `async for item in spider.stream()` mit Echtzeit-Statistiken -- ideal für UI, Pipelines und lang laufende Crawls.
+- 🛡️ **Erkennung blockierter Anfragen**: Automatische Erkennung und Wiederholung blockierter Anfragen mit anpassbarer Logik.
+- 📦 **Integrierter Export**: Ergebnisse über Hooks und Ihre eigene Pipeline oder den integrierten JSON/JSONL-Export mit `result.items.to_json()` / `result.items.to_jsonl()` exportieren.
+
+### Erweitertes Website-Abrufen mit Session-Unterstützung
+- **HTTP-Anfragen**: Schnelle und heimliche HTTP-Anfragen mit der `Fetcher`-Klasse. Kann Browser-TLS-Fingerprints und Header imitieren und HTTP/3 verwenden.
+- **Dynamisches Laden**: Dynamische Websites mit vollständiger Browser-Automatisierung über die `DynamicFetcher`-Klasse abrufen, die Playwrights Chromium und Google Chrome unterstützt.
+- **Anti-Bot-Umgehung**: Erweiterte Stealth-Fähigkeiten mit `StealthyFetcher` und Fingerprint-Spoofing. Kann alle Arten von Cloudflares Turnstile/Interstitial einfach mit Automatisierung umgehen.
+- **Session-Verwaltung**: Persistente Session-Unterstützung mit den Klassen `FetcherSession`, `StealthySession` und `DynamicSession` für Cookie- und Zustandsverwaltung über Anfragen hinweg.
+- **Proxy-Rotation**: Integrierter `ProxyRotator` mit zyklischen oder benutzerdefinierten Rotationsstrategien über alle Session-Typen hinweg, plus Proxy-Überschreibungen pro Anfrage.
+- **Domain-Blockierung**: Anfragen an bestimmte Domains (und deren Subdomains) in browserbasierten Fetchern blockieren.
+- **Async-Unterstützung**: Vollständige async-Unterstützung über alle Fetcher und dedizierte async Session-Klassen hinweg.
+
+### Adaptives Scraping & KI-Integration
+- 🔄 **Intelligente Element-Verfolgung**: Elemente nach Website-Änderungen mit intelligenten Ähnlichkeitsalgorithmen neu lokalisieren.
+- 🎯 **Intelligente flexible Auswahl**: CSS-Selektoren, XPath-Selektoren, filterbasierte Suche, Textsuche, Regex-Suche und mehr.
+- 🔍 **Ähnliche Elemente finden**: Elemente, die gefundenen Elementen ähnlich sind, automatisch lokalisieren.
+- 🤖 **MCP-Server für die Verwendung mit KI**: Integrierter MCP-Server für KI-unterstütztes Web Scraping und Datenextraktion. Der MCP-Server verfügt über leistungsstarke, benutzerdefinierte Funktionen, die Scrapling nutzen, um gezielten Inhalt zu extrahieren, bevor er an die KI (Claude/Cursor/etc.) übergeben wird, wodurch Vorgänge beschleunigt und Kosten durch Minimierung der Token-Nutzung gesenkt werden. ([Demo-Video](https://www.youtube.com/watch?v=qyFk3ZNwOxE))
+
+### Hochleistungs- und praxiserprobte Architektur
+- 🚀 **Blitzschnell**: Optimierte Leistung, die die meisten Python-Scraping-Bibliotheken übertrifft.
+- 🔋 **Speichereffizient**: Optimierte Datenstrukturen und Lazy Loading für einen minimalen Speicher-Footprint.
+- ⚡ **Schnelle JSON-Serialisierung**: 10x schneller als die Standardbibliothek.
+- 🏗️ **Praxiserprobt**: Scrapling hat nicht nur eine Testabdeckung von 92% und eine vollständige Type-Hints-Abdeckung, sondern wird seit dem letzten Jahr täglich von Hunderten von Web Scrapern verwendet.
+
+### Entwickler-/Web-Scraper-freundliche Erfahrung
+- 🎯 **Interaktive Web-Scraping-Shell**: Optionale integrierte IPython-Shell mit Scrapling-Integration, Shortcuts und neuen Tools zur Beschleunigung der Web-Scraping-Skriptentwicklung, wie das Konvertieren von Curl-Anfragen in Scrapling-Anfragen und das Anzeigen von Anfrageergebnissen in Ihrem Browser.
+- 🚀 **Direkt vom Terminal aus verwenden**: Optional können Sie Scrapling verwenden, um eine URL zu scrapen, ohne eine einzige Codezeile zu schreiben!
+- 🛠️ **Umfangreiche Navigations-API**: Erweiterte DOM-Traversierung mit Eltern-, Geschwister- und Kind-Navigationsmethoden.
+- 🧬 **Verbesserte Textverarbeitung**: Integrierte Regex, Bereinigungsmethoden und optimierte String-Operationen.
+- 📝 **Automatische Selektorgenerierung**: Robuste CSS/XPath-Selektoren für jedes Element generieren.
+- 🔌 **Vertraute API**: Ähnlich wie Scrapy/BeautifulSoup mit denselben Pseudo-Elementen, die in Scrapy/Parsel verwendet werden.
+- 📘 **Vollständige Typabdeckung**: Vollständige Type Hints für hervorragende IDE-Unterstützung und Code-Vervollständigung. Die gesamte Codebasis wird bei jeder Änderung automatisch mit **PyRight** und **MyPy** gescannt.
+- 🔋 **Fertiges Docker-Image**: Mit jeder Veröffentlichung wird automatisch ein Docker-Image erstellt und gepusht, das alle Browser enthält.
+
+## Erste Schritte
+
+Hier ein kurzer Überblick über das, was Scrapling kann, ohne zu sehr ins Detail zu gehen.
+
+### Grundlegende Verwendung
+HTTP-Anfragen mit Session-Unterstützung
+```python
+from scrapling.fetchers import Fetcher, FetcherSession
+
+with FetcherSession(impersonate='chrome') as session:  # Neueste Version von Chromes TLS-Fingerprint verwenden
+    page = session.get('https://quotes.toscrape.com/', stealthy_headers=True)
+    quotes = page.css('.quote .text::text').getall()
+
+# Oder einmalige Anfragen verwenden
+page = Fetcher.get('https://quotes.toscrape.com/')
+quotes = page.css('.quote .text::text').getall()
+```
+Erweiterter Stealth-Modus
+```python
+from scrapling.fetchers import StealthyFetcher, StealthySession
+
+with StealthySession(headless=True, solve_cloudflare=True) as session:  # Browser offen halten, bis Sie fertig sind
+    page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False)
+    data = page.css('#padded_content a').getall()
+
+# Oder einmaligen Anfragenstil verwenden: öffnet den Browser für diese Anfrage und schließt ihn nach Abschluss
+page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare')
+data = page.css('#padded_content a').getall()
+```
+Vollständige Browser-Automatisierung
+```python
+from scrapling.fetchers import DynamicFetcher, DynamicSession
+
+with DynamicSession(headless=True, disable_resources=False, network_idle=True) as session:  # Browser offen halten, bis Sie fertig sind
+    page = session.fetch('https://quotes.toscrape.com/', load_dom=False)
+    data = page.xpath('//span[@class="text"]/text()').getall()  # XPath-Selektor, falls bevorzugt
+
+# Oder einmaligen Anfragenstil verwenden: öffnet den Browser für diese Anfrage und schließt ihn nach Abschluss
+page = DynamicFetcher.fetch('https://quotes.toscrape.com/')
+data = page.css('.quote .text::text').getall()
+```
+
+### Spiders
+Vollständige Crawler mit parallelen Anfragen, mehreren Session-Typen und Pause & Resume erstellen:
+```python
+from scrapling.spiders import Spider, Request, Response
+
+class QuotesSpider(Spider):
+    name = "quotes"
+    start_urls = ["https://quotes.toscrape.com/"]
+    concurrent_requests = 10
+
+    async def parse(self, response: Response):
+        for quote in response.css('.quote'):
+            yield {
+                "text": quote.css('.text::text').get(),
+                "author": quote.css('.author::text').get(),
+            }
+
+        next_page = response.css('.next a')
+        if next_page:
+            yield response.follow(next_page[0].attrib['href'])
+
+result = QuotesSpider().start()
+print(f"{len(result.items)} Zitate gescrapt")
+result.items.to_json("quotes.json")
+```
+Mehrere Session-Typen in einem einzigen Spider verwenden:
+```python
+from scrapling.spiders import Spider, Request, Response
+from scrapling.fetchers import FetcherSession, AsyncStealthySession
+
+class MultiSessionSpider(Spider):
+    name = "multi"
+    start_urls = ["https://example.com/"]
+
+    def configure_sessions(self, manager):
+        manager.add("fast", FetcherSession(impersonate="chrome"))
+        manager.add("stealth", AsyncStealthySession(headless=True), lazy=True)
+
+    async def parse(self, response: Response):
+        for link in response.css('a::attr(href)').getall():
+            # Geschützte Seiten über die Stealth-Session leiten
+            if "protected" in link:
+                yield Request(link, sid="stealth")
+            else:
+                yield Request(link, sid="fast", callback=self.parse)  # Expliziter Callback
+```
+Lange Crawls mit Checkpoints pausieren und fortsetzen, indem Sie den Spider so starten:
+```python
+QuotesSpider(crawldir="./crawl_data").start()
+```
+Drücken Sie Strg+C, um kontrolliert zu pausieren -- der Fortschritt wird automatisch gespeichert. Wenn Sie den Spider später erneut starten, übergeben Sie dasselbe `crawldir`, und er setzt dort fort, wo er aufgehört hat.
+
+### Erweitertes Parsing & Navigation
+```python
+from scrapling.fetchers import Fetcher
+
+# Umfangreiche Elementauswahl und Navigation
+page = Fetcher.get('https://quotes.toscrape.com/')
+
+# Zitate mit verschiedenen Auswahlmethoden abrufen
+quotes = page.css('.quote')  # CSS-Selektor
+quotes = page.xpath('//div[@class="quote"]')  # XPath
+quotes = page.find_all('div', {'class': 'quote'})  # BeautifulSoup-Stil
+# Gleich wie
+quotes = page.find_all('div', class_='quote')
+quotes = page.find_all(['div'], class_='quote')
+quotes = page.find_all(class_='quote')  # und so weiter...
+# Element nach Textinhalt finden
+quotes = page.find_by_text('quote', tag='div')
+
+# Erweiterte Navigation
+quote_text = page.css('.quote')[0].css('.text::text').get()
+quote_text = page.css('.quote').css('.text::text').getall()  # Verkettete Selektoren
+first_quote = page.css('.quote')[0]
+author = first_quote.next_sibling.css('.author::text')
+parent_container = first_quote.parent
+
+# Elementbeziehungen und Ähnlichkeit
+similar_elements = first_quote.find_similar()
+below_elements = first_quote.below_elements()
+```
+Sie können den Parser direkt verwenden, wenn Sie keine Websites abrufen möchten, wie unten gezeigt:
+```python
+from scrapling.parser import Selector
+
+page = Selector("<html>...</html>")
+```
+Und es funktioniert genau auf die gleiche Weise!
+
+### Beispiele für async Session-Verwaltung
+```python
+import asyncio
+from scrapling.fetchers import FetcherSession, AsyncStealthySession, AsyncDynamicSession
+
+async with FetcherSession(http3=True) as session:  # `FetcherSession` ist kontextbewusst und kann sowohl in sync- als auch in async-Mustern arbeiten
+    page1 = session.get('https://quotes.toscrape.com/')
+    page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135')
+
+# Async-Session-Verwendung
+async with AsyncStealthySession(max_pages=2) as session:
+    tasks = []
+    urls = ['https://example.com/page1', 'https://example.com/page2']
+
+    for url in urls:
+        task = session.fetch(url)
+        tasks.append(task)
+
+    print(session.get_pool_stats())  # Optional - Der Status des Browser-Tab-Pools (beschäftigt/frei/Fehler)
+    results = await asyncio.gather(*tasks)
+    print(session.get_pool_stats())
+```
+
+## CLI & Interaktive Shell
+
+Scrapling enthält eine leistungsstarke Befehlszeilenschnittstelle:
+
+[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339)
+
+Interaktive Web-Scraping-Shell starten
+```bash
+scrapling shell
+```
+Seiten direkt ohne Programmierung in eine Datei extrahieren (extrahiert standardmäßig den Inhalt im `body`-Tag). Wenn die Ausgabedatei mit `.txt` endet, wird der Textinhalt des Ziels extrahiert. Wenn sie mit `.md` endet, ist es eine Markdown-Darstellung des HTML-Inhalts; wenn sie mit `.html` endet, ist es der HTML-Inhalt selbst.
+```bash
+scrapling extract get 'https://example.com' content.md
+scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome'  # Alle Elemente, die dem CSS-Selektor '#fromSkipToProducts' entsprechen
+scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless
+scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare
+```
+
+> [!NOTE]
+> Es gibt viele zusätzliche Funktionen, aber wir möchten diese Seite prägnant halten, einschließlich des MCP-Servers und der interaktiven Web-Scraping-Shell. Schauen Sie sich die vollständige Dokumentation [hier](https://scrapling.readthedocs.io/en/latest/) an
+
+## Leistungsbenchmarks
+
+Scrapling ist nicht nur leistungsstark -- es ist auch blitzschnell. Die folgenden Benchmarks vergleichen Scraplings Parser mit den neuesten Versionen anderer beliebter Bibliotheken.
+
+### Textextraktions-Geschwindigkeitstest (5000 verschachtelte Elemente)
+
+| # |    Bibliothek     | Zeit (ms) | vs Scrapling |
+|---|:-----------------:|:---------:|:------------:|
+| 1 |     Scrapling     |   2.02    |     1.0x     |
+| 2 |   Parsel/Scrapy   |   2.04    |     1.01     |
+| 3 |     Raw Lxml      |   2.54    |    1.257     |
+| 4 |      PyQuery      |   24.17   |     ~12x     |
+| 5 |    Selectolax     |   82.63   |     ~41x     |
+| 6 |  MechanicalSoup   |  1549.71  |   ~767.1x    |
+| 7 |   BS4 with Lxml   |  1584.31  |   ~784.3x    |
+| 8 | BS4 with html5lib |  3391.91  |   ~1679.1x   |
+
+
+### Element-Ähnlichkeit & Textsuche-Leistung
+
+Scraplings adaptive Element-Finding-Fähigkeiten übertreffen Alternativen deutlich:
+
+| Bibliothek  | Zeit (ms) | vs Scrapling |
+|-------------|:---------:|:------------:|
+| Scrapling   |   2.39    |     1.0x     |
+| AutoScraper |   12.45   |    5.209x    |
+
+
+> Alle Benchmarks stellen Durchschnittswerte von über 100 Durchläufen dar. Siehe [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) für die Methodik.
+
+## Installation
+
+Scrapling erfordert Python 3.10 oder höher:
+
+```bash
+pip install scrapling
+```
+
+Diese Installation enthält nur die Parser-Engine und ihre Abhängigkeiten, ohne Fetcher oder Kommandozeilenabhängigkeiten.
+
+### Optionale Abhängigkeiten
+
+1. Wenn Sie eine der folgenden zusätzlichen Funktionen, die Fetcher oder ihre Klassen verwenden möchten, müssen Sie die Abhängigkeiten der Fetcher und ihre Browser-Abhängigkeiten wie folgt installieren:
+    ```bash
+    pip install "scrapling[fetchers]"
+
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
+    ```
+
+    Dies lädt alle Browser zusammen mit ihren Systemabhängigkeiten und Fingerprint-Manipulationsabhängigkeiten herunter.
+
+    Oder Sie können sie aus dem Code heraus installieren, anstatt einen Befehl auszuführen:
+    ```python
+    from scrapling.cli import install
+
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
+2. Zusätzliche Funktionen:
+   - MCP-Server-Funktion installieren:
+       ```bash
+       pip install "scrapling[ai]"
+       ```
+   - Shell-Funktionen installieren (Web-Scraping-Shell und der `extract`-Befehl):
+       ```bash
+       pip install "scrapling[shell]"
+       ```
+   - Alles installieren:
+       ```bash
+       pip install "scrapling[all]"
+       ```
+   Denken Sie daran, dass Sie nach einem dieser Extras (falls noch nicht geschehen) die Browser-Abhängigkeiten mit `scrapling install` installieren müssen
+
+### Docker
+Sie können auch ein Docker-Image mit allen Extras und Browsern mit dem folgenden Befehl von DockerHub installieren:
+```bash
+docker pull pyd4vinci/scrapling
+```
+Oder laden Sie es aus der GitHub-Registry herunter:
+```bash
+docker pull ghcr.io/d4vinci/scrapling:latest
+```
+Dieses Image wird automatisch mit GitHub Actions und dem Hauptzweig des Repositorys erstellt und gepusht.
+
+## Beitragen
+
+Wir freuen uns über Beiträge! Bitte lesen Sie unsere [Beitragsrichtlinien](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md), bevor Sie beginnen.
+
+## Haftungsausschluss
+
+> [!CAUTION]
+> Diese Bibliothek wird nur zu Bildungs- und Forschungszwecken bereitgestellt. Durch die Nutzung dieser Bibliothek erklären Sie sich damit einverstanden, lokale und internationale Gesetze zum Daten-Scraping und Datenschutz einzuhalten. Die Autoren und Mitwirkenden sind nicht verantwortlich für Missbrauch dieser Software. Respektieren Sie immer die Nutzungsbedingungen von Websites und robots.txt-Dateien.
+
+## Lizenz
+
+Diese Arbeit ist unter der BSD-3-Clause-Lizenz lizenziert.
+
+## Danksagungen
+
+Dieses Projekt enthält angepassten Code von:
+- Parsel (BSD-Lizenz) -- Verwendet für das [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py)-Submodul
+
+---
+<div align="center"><small>Entworfen und hergestellt mit ❤️ von Karim Shoair.</small></div><br>

docs/README_ES.md

@@ -0,0 +1,426 @@
+<!-- mcp-name: io.github.D4Vinci/Scrapling -->
+
+<h1 align="center">
+    <a href="https://scrapling.readthedocs.io">
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_dark.svg?sanitize=true">
+          <img alt="Scrapling Poster" src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_light.svg?sanitize=true">
+        </picture>
+    </a>
+    <br>
+    <small>Effortless Web Scraping for the Modern Web</small>
+</h1>
+
+<p align="center">
+    <a href="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml" alt="Tests">
+        <img alt="Tests" src="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg"></a>
+    <a href="https://badge.fury.io/py/Scrapling" alt="PyPI version">
+        <img alt="PyPI version" src="https://badge.fury.io/py/Scrapling.svg"></a>
+    <a href="https://pepy.tech/project/scrapling" alt="PyPI Downloads">
+        <img alt="PyPI Downloads" src="https://static.pepy.tech/personalized-badge/scrapling?period=total&units=INTERNATIONAL_SYSTEM&left_color=GREY&right_color=GREEN&left_text=Downloads"></a>
+    <br/>
+    <a href="https://discord.gg/EMgGbDceNQ" alt="Discord" target="_blank">
+      <img alt="Discord" src="https://img.shields.io/discord/1360786381042880532?style=social&logo=discord&link=https%3A%2F%2Fdiscord.gg%2FEMgGbDceNQ">
+    </a>
+    <a href="https://x.com/Scrapling_dev" alt="X (formerly Twitter)">
+      <img alt="X (formerly Twitter) Follow" src="https://img.shields.io/twitter/follow/Scrapling_dev?style=social&logo=x&link=https%3A%2F%2Fx.com%2FScrapling_dev">
+    </a>
+    <br/>
+    <a href="https://pypi.org/project/scrapling/" alt="Supported Python versions">
+        <img alt="Supported Python versions" src="https://img.shields.io/pypi/pyversions/scrapling.svg"></a>
+</p>
+
+<p align="center">
+    <a href="https://scrapling.readthedocs.io/en/latest/parsing/selection/"><strong>Métodos de selección</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/fetching/choosing/"><strong>Elegir un fetcher</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/architecture.html"><strong>Spiders</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/proxy-blocking.html"><strong>Rotación de proxy</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/cli/overview/"><strong>CLI</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/ai/mcp-server/"><strong>Modo MCP</strong></a>
+</p>
+
+Scrapling es un framework de Web Scraping adaptativo que se encarga de todo, desde una sola solicitud hasta un rastreo a gran escala.
+
+Su parser aprende de los cambios de los sitios web y relocaliza automáticamente tus elementos cuando las páginas se actualizan. Sus fetchers evaden sistemas anti-bot como Cloudflare Turnstile de forma nativa. Y su framework Spider te permite escalar a rastreos concurrentes con múltiples sesiones, con Pause & Resume y rotación automática de Proxy, todo en unas pocas líneas de Python. Una biblioteca, cero compromisos.
+
+Rastreos ultrarrápidos con estadísticas en tiempo real y Streaming. Construido por Web Scrapers para Web Scrapers y usuarios regulares, hay algo para todos.
+
+```python
+from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
+StealthyFetcher.adaptive = True
+p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True)  # ¡Obtén el sitio web bajo el radar!
+products = p.css('.product', auto_save=True)                                        # ¡Extrae datos que sobreviven a cambios de diseño del sitio web!
+products = p.css('.product', adaptive=True)                                         # Más tarde, si la estructura del sitio web cambia, ¡pasa `adaptive=True` para encontrarlos!
+```
+O escala a rastreos completos
+```python
+from scrapling.spiders import Spider, Response
+
+class MySpider(Spider):
+  name = "demo"
+  start_urls = ["https://example.com/"]
+
+  async def parse(self, response: Response):
+      for item in response.css('.product'):
+          yield {"title": item.css('h2::text').get()}
+
+MySpider().start()
+```
+
+
+# Patrocinadores Platino
+
+<i><sub>¿Quieres ser la primera empresa en aparecer aquí? Haz clic [aquí](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
+# Patrocinadores
+
+<!-- sponsors -->
+
+<a href="https://www.scrapeless.com/en?utm_source=official&utm_term=scrapling" target="_blank" title="Effortless Web Scraping Toolkit for Business and Developers"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/scrapeless.jpg"></a>
+<a href="https://www.thordata.com/?ls=github&lk=github" target="_blank" title="Unblockable proxies and scraping infrastructure, delivering real-time, reliable web data to power AI models and workflows."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/thordata.jpg"></a>
+<a href="https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling" target="_blank" title="Evomi is your Swiss Quality Proxy Provider, starting at $0.49/GB"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/evomi.png"></a>
+<a href="https://serpapi.com/?utm_source=scrapling" target="_blank" title="Scrape Google and other search engines with SerpApi"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SerpApi.png"></a>
+<a href="https://visit.decodo.com/Dy6W0b" target="_blank" title="Try the Most Efficient Residential Proxies for Free"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/decodo.png"></a>
+<a href="https://petrosky.io/d4vinci" target="_blank" title="PetroSky delivers cutting-edge VPS hosting."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/petrosky.png"></a>
+<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
+<a href="https://proxyempire.io/" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
+<a href="https://hypersolutions.co/?utm_source=github&utm_medium=readme&utm_campaign=scrapling" target="_blank" title="Bot Protection Bypass API for Akamai, DataDome, Incapsula & Kasada"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/HyperSolutions.png"></a>
+
+
+<a href="https://www.swiftproxy.net/" target="_blank" title="Unlock Reliable Proxy Services with Swiftproxy!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/swiftproxy.png"></a>
+<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
+<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
+
+<!-- /sponsors -->
+
+<i><sub>¿Quieres mostrar tu anuncio aquí? ¡Haz clic [aquí](https://github.com/sponsors/D4Vinci) y elige el nivel que te convenga!</sub></i>
+
+---
+
+## Características Principales
+
+### Spiders — Un Framework Completo de Rastreo
+- 🕷️ **API de Spider al estilo Scrapy**: Define spiders con `start_urls`, callbacks async `parse`, y objetos `Request`/`Response`.
+- ⚡ **Rastreo Concurrente**: Límites de concurrencia configurables, limitación por dominio y retrasos de descarga.
+- 🔄 **Soporte Multi-Session**: Interfaz unificada para solicitudes HTTP y navegadores headless sigilosos en un solo Spider — enruta solicitudes a diferentes sesiones por ID.
+- 💾 **Pause & Resume**: Persistencia de rastreo basada en Checkpoint. Presiona Ctrl+C para un cierre ordenado; reinicia para continuar desde donde lo dejaste.
+- 📡 **Modo Streaming**: Transmite elementos extraídos a medida que llegan con `async for item in spider.stream()` con estadísticas en tiempo real — ideal para UI, pipelines y rastreos de larga duración.
+- 🛡️ **Detección de Solicitudes Bloqueadas**: Detección automática y reintento de solicitudes bloqueadas con lógica personalizable.
+- 📦 **Exportación Integrada**: Exporta resultados a través de hooks y tu propio pipeline o el JSON/JSONL integrado con `result.items.to_json()` / `result.items.to_jsonl()` respectivamente.
+
+### Obtención Avanzada de Sitios Web con Soporte de Session
+- **Solicitudes HTTP**: Solicitudes HTTP rápidas y sigilosas con la clase `Fetcher`. Puede imitar el fingerprint TLS de los navegadores, encabezados y usar HTTP/3.
+- **Carga Dinámica**: Obtén sitios web dinámicos con automatización completa del navegador a través de la clase `DynamicFetcher` compatible con Chromium de Playwright y Google Chrome.
+- **Evasión Anti-bot**: Capacidades de sigilo avanzadas con `StealthyFetcher` y falsificación de fingerprint. Puede evadir fácilmente todos los tipos de Turnstile/Interstitial de Cloudflare con automatización.
+- **Gestión de Session**: Soporte de sesión persistente con las clases `FetcherSession`, `StealthySession` y `DynamicSession` para la gestión de cookies y estado entre solicitudes.
+- **Rotación de Proxy**: `ProxyRotator` integrado con estrategias de rotación cíclica o personalizadas en todos los tipos de sesión, además de sobrescrituras de Proxy por solicitud.
+- **Bloqueo de Dominios**: Bloquea solicitudes a dominios específicos (y sus subdominios) en fetchers basados en navegador.
+- **Soporte Async**: Soporte async completo en todos los fetchers y clases de sesión async dedicadas.
+
+### Scraping Adaptativo e Integración con IA
+- 🔄 **Seguimiento Inteligente de Elementos**: Relocaliza elementos después de cambios en el sitio web usando algoritmos inteligentes de similitud.
+- 🎯 **Selección Flexible Inteligente**: Selectores CSS, selectores XPath, búsqueda basada en filtros, búsqueda de texto, búsqueda regex y más.
+- 🔍 **Encontrar Elementos Similares**: Localiza automáticamente elementos similares a los elementos encontrados.
+- 🤖 **Servidor MCP para usar con IA**: Servidor MCP integrado para Web Scraping asistido por IA y extracción de datos. El servidor MCP presenta capacidades potentes y personalizadas que aprovechan Scrapling para extraer contenido específico antes de pasarlo a la IA (Claude/Cursor/etc), acelerando así las operaciones y reduciendo costos al minimizar el uso de tokens. ([video demo](https://www.youtube.com/watch?v=qyFk3ZNwOxE))
+
+### Arquitectura de Alto Rendimiento y Probada en Batalla
+- 🚀 **Ultrarrápido**: Rendimiento optimizado que supera a la mayoría de las bibliotecas de Web Scraping de Python.
+- 🔋 **Eficiente en Memoria**: Estructuras de datos optimizadas y carga diferida para una huella de memoria mínima.
+- ⚡ **Serialización JSON Rápida**: 10 veces más rápido que la biblioteca estándar.
+- 🏗️ **Probado en batalla**: Scrapling no solo tiene una cobertura de pruebas del 92% y cobertura completa de type hints, sino que ha sido utilizado diariamente por cientos de Web Scrapers durante el último año.
+
+### Experiencia Amigable para Desarrolladores/Web Scrapers
+- 🎯 **Shell Interactivo de Web Scraping**: Shell IPython integrado opcional con integración de Scrapling, atajos y nuevas herramientas para acelerar el desarrollo de scripts de Web Scraping, como convertir solicitudes curl a solicitudes Scrapling y ver resultados de solicitudes en tu navegador.
+- 🚀 **Úsalo directamente desde la Terminal**: Opcionalmente, ¡puedes usar Scrapling para hacer scraping de una URL sin escribir ni una sola línea de código!
+- 🛠️ **API de Navegación Rica**: Recorrido avanzado del DOM con métodos de navegación de padres, hermanos e hijos.
+- 🧬 **Procesamiento de Texto Mejorado**: Métodos integrados de regex, limpieza y operaciones de cadena optimizadas.
+- 📝 **Generación Automática de Selectores**: Genera selectores CSS/XPath robustos para cualquier elemento.
+- 🔌 **API Familiar**: Similar a Scrapy/BeautifulSoup con los mismos pseudo-elementos usados en Scrapy/Parsel.
+- 📘 **Cobertura Completa de Tipos**: Type hints completos para excelente soporte de IDE y autocompletado de código. Todo el código fuente se escanea automáticamente con **PyRight** y **MyPy** en cada cambio.
+- 🔋 **Imagen Docker Lista**: Con cada lanzamiento, se construye y publica automáticamente una imagen Docker que contiene todos los navegadores.
+
+## Primeros Pasos
+
+Aquí tienes un vistazo rápido de lo que Scrapling puede hacer sin entrar en profundidad.
+
+### Uso Básico
+Solicitudes HTTP con soporte de sesión
+```python
+from scrapling.fetchers import Fetcher, FetcherSession
+
+with FetcherSession(impersonate='chrome') as session:  # Usa la última versión del fingerprint TLS de Chrome
+    page = session.get('https://quotes.toscrape.com/', stealthy_headers=True)
+    quotes = page.css('.quote .text::text').getall()
+
+# O usa solicitudes de una sola vez
+page = Fetcher.get('https://quotes.toscrape.com/')
+quotes = page.css('.quote .text::text').getall()
+```
+Modo sigiloso avanzado
+```python
+from scrapling.fetchers import StealthyFetcher, StealthySession
+
+with StealthySession(headless=True, solve_cloudflare=True) as session:  # Mantén el navegador abierto hasta que termines
+    page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False)
+    data = page.css('#padded_content a').getall()
+
+# O usa el estilo de solicitud de una sola vez, abre el navegador para esta solicitud, luego lo cierra después de terminar
+page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare')
+data = page.css('#padded_content a').getall()
+```
+Automatización completa del navegador
+```python
+from scrapling.fetchers import DynamicFetcher, DynamicSession
+
+with DynamicSession(headless=True, disable_resources=False, network_idle=True) as session:  # Mantén el navegador abierto hasta que termines
+    page = session.fetch('https://quotes.toscrape.com/', load_dom=False)
+    data = page.xpath('//span[@class="text"]/text()').getall()  # Selector XPath si lo prefieres
+
+# O usa el estilo de solicitud de una sola vez, abre el navegador para esta solicitud, luego lo cierra después de terminar
+page = DynamicFetcher.fetch('https://quotes.toscrape.com/')
+data = page.css('.quote .text::text').getall()
+```
+
+### Spiders
+Construye rastreadores completos con solicitudes concurrentes, múltiples tipos de sesión y Pause & Resume:
+```python
+from scrapling.spiders import Spider, Request, Response
+
+class QuotesSpider(Spider):
+    name = "quotes"
+    start_urls = ["https://quotes.toscrape.com/"]
+    concurrent_requests = 10
+
+    async def parse(self, response: Response):
+        for quote in response.css('.quote'):
+            yield {
+                "text": quote.css('.text::text').get(),
+                "author": quote.css('.author::text').get(),
+            }
+
+        next_page = response.css('.next a')
+        if next_page:
+            yield response.follow(next_page[0].attrib['href'])
+
+result = QuotesSpider().start()
+print(f"Se extrajeron {len(result.items)} citas")
+result.items.to_json("quotes.json")
+```
+Usa múltiples tipos de sesión en un solo Spider:
+```python
+from scrapling.spiders import Spider, Request, Response
+from scrapling.fetchers import FetcherSession, AsyncStealthySession
+
+class MultiSessionSpider(Spider):
+    name = "multi"
+    start_urls = ["https://example.com/"]
+
+    def configure_sessions(self, manager):
+        manager.add("fast", FetcherSession(impersonate="chrome"))
+        manager.add("stealth", AsyncStealthySession(headless=True), lazy=True)
+
+    async def parse(self, response: Response):
+        for link in response.css('a::attr(href)').getall():
+            # Enruta las páginas protegidas a través de la sesión sigilosa
+            if "protected" in link:
+                yield Request(link, sid="stealth")
+            else:
+                yield Request(link, sid="fast", callback=self.parse)  # callback explícito
+```
+Pausa y reanuda rastreos largos con checkpoints ejecutando el Spider así:
+```python
+QuotesSpider(crawldir="./crawl_data").start()
+```
+Presiona Ctrl+C para pausar de forma ordenada — el progreso se guarda automáticamente. Después, cuando inicies el Spider de nuevo, pasa el mismo `crawldir`, y continuará desde donde se detuvo.
+
+### Análisis Avanzado y Navegación
+```python
+from scrapling.fetchers import Fetcher
+
+# Selección rica de elementos y navegación
+page = Fetcher.get('https://quotes.toscrape.com/')
+
+# Obtén citas con múltiples métodos de selección
+quotes = page.css('.quote')  # Selector CSS
+quotes = page.xpath('//div[@class="quote"]')  # XPath
+quotes = page.find_all('div', {'class': 'quote'})  # Estilo BeautifulSoup
+# Igual que
+quotes = page.find_all('div', class_='quote')
+quotes = page.find_all(['div'], class_='quote')
+quotes = page.find_all(class_='quote')  # y así sucesivamente...
+# Encuentra elementos por contenido de texto
+quotes = page.find_by_text('quote', tag='div')
+
+# Navegación avanzada
+quote_text = page.css('.quote')[0].css('.text::text').get()
+quote_text = page.css('.quote').css('.text::text').getall()  # Selectores encadenados
+first_quote = page.css('.quote')[0]
+author = first_quote.next_sibling.css('.author::text')
+parent_container = first_quote.parent
+
+# Relaciones y similitud de elementos
+similar_elements = first_quote.find_similar()
+below_elements = first_quote.below_elements()
+```
+Puedes usar el parser directamente si no necesitas obtener sitios web, como se muestra a continuación:
+```python
+from scrapling.parser import Selector
+
+page = Selector("<html>...</html>")
+```
+¡Y funciona exactamente de la misma manera!
+
+### Ejemplos de Gestión de Session Async
+```python
+import asyncio
+from scrapling.fetchers import FetcherSession, AsyncStealthySession, AsyncDynamicSession
+
+async with FetcherSession(http3=True) as session:  # `FetcherSession` es consciente del contexto y puede funcionar tanto en patrones sync/async
+    page1 = session.get('https://quotes.toscrape.com/')
+    page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135')
+
+# Uso de sesión async
+async with AsyncStealthySession(max_pages=2) as session:
+    tasks = []
+    urls = ['https://example.com/page1', 'https://example.com/page2']
+
+    for url in urls:
+        task = session.fetch(url)
+        tasks.append(task)
+
+    print(session.get_pool_stats())  # Opcional - El estado del pool de pestañas del navegador (ocupado/libre/error)
+    results = await asyncio.gather(*tasks)
+    print(session.get_pool_stats())
+```
+
+## CLI y Shell Interactivo
+
+Scrapling incluye una poderosa interfaz de línea de comandos:
+
+[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339)
+
+Lanzar el Shell interactivo de Web Scraping
+```bash
+scrapling shell
+```
+Extraer páginas a un archivo directamente sin programar (Extrae el contenido dentro de la etiqueta `body` por defecto). Si el archivo de salida termina con `.txt`, entonces se extraerá el contenido de texto del objetivo. Si termina con `.md`, será una representación Markdown del contenido HTML; si termina con `.html`, será el contenido HTML en sí mismo.
+```bash
+scrapling extract get 'https://example.com' content.md
+scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome'  # Todos los elementos que coinciden con el selector CSS '#fromSkipToProducts'
+scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless
+scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare
+```
+
+> [!NOTE]
+> Hay muchas características adicionales, pero queremos mantener esta página concisa, incluyendo el servidor MCP y el Shell Interactivo de Web Scraping. Consulta la documentación completa [aquí](https://scrapling.readthedocs.io/en/latest/)
+
+## Benchmarks de Rendimiento
+
+Scrapling no solo es potente, también es ultrarrápido. Los siguientes benchmarks comparan el parser de Scrapling con las últimas versiones de otras bibliotecas populares.
+
+### Prueba de Velocidad de Extracción de Texto (5000 elementos anidados)
+
+| # |    Biblioteca     | Tiempo (ms) | vs Scrapling |
+|---|:-----------------:|:-----------:|:------------:|
+| 1 |     Scrapling     |    2.02     |     1.0x     |
+| 2 |   Parsel/Scrapy   |    2.04     |     1.01     |
+| 3 |     Raw Lxml      |    2.54     |    1.257     |
+| 4 |      PyQuery      |    24.17    |     ~12x     |
+| 5 |    Selectolax     |    82.63    |     ~41x     |
+| 6 |  MechanicalSoup   |   1549.71   |   ~767.1x    |
+| 7 |   BS4 with Lxml   |   1584.31   |   ~784.3x    |
+| 8 | BS4 with html5lib |   3391.91   |   ~1679.1x   |
+
+
+### Rendimiento de Similitud de Elementos y Búsqueda de Texto
+
+Las capacidades de búsqueda adaptativa de elementos de Scrapling superan significativamente a las alternativas:
+
+| Biblioteca  | Tiempo (ms) | vs Scrapling |
+|-------------|:-----------:|:------------:|
+| Scrapling   |    2.39     |     1.0x     |
+| AutoScraper |    12.45    |    5.209x    |
+
+
+> Todos los benchmarks representan promedios de más de 100 ejecuciones. Ver [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) para la metodología.
+
+## Instalación
+
+Scrapling requiere Python 3.10 o superior:
+
+```bash
+pip install scrapling
+```
+
+Esta instalación solo incluye el motor de análisis y sus dependencias, sin ningún fetcher ni dependencias de línea de comandos.
+
+### Dependencias Opcionales
+
+1. Si vas a usar alguna de las características adicionales a continuación, los fetchers, o sus clases, necesitarás instalar las dependencias de los fetchers y sus dependencias del navegador de la siguiente manera:
+    ```bash
+    pip install "scrapling[fetchers]"
+
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
+    ```
+
+    Esto descarga todos los navegadores, junto con sus dependencias del sistema y dependencias de manipulación de fingerprint.
+
+    O puedes instalarlos desde el código en lugar de ejecutar un comando:
+    ```python
+    from scrapling.cli import install
+
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
+2. Características adicionales:
+   - Instalar la característica del servidor MCP:
+       ```bash
+       pip install "scrapling[ai]"
+       ```
+   - Instalar características del Shell (Shell de Web Scraping y el comando `extract`):
+       ```bash
+       pip install "scrapling[shell]"
+       ```
+   - Instalar todo:
+       ```bash
+       pip install "scrapling[all]"
+       ```
+   Recuerda que necesitas instalar las dependencias del navegador con `scrapling install` después de cualquiera de estos extras (si no lo hiciste ya)
+
+### Docker
+También puedes instalar una imagen Docker con todos los extras y navegadores con el siguiente comando desde DockerHub:
+```bash
+docker pull pyd4vinci/scrapling
+```
+O descárgala desde el registro de GitHub:
+```bash
+docker pull ghcr.io/d4vinci/scrapling:latest
+```
+Esta imagen se construye y publica automáticamente usando GitHub Actions y la rama principal del repositorio.
+
+## Contribuir
+
+¡Damos la bienvenida a las contribuciones! Por favor lee nuestras [pautas de contribución](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md) antes de comenzar.
+
+## Descargo de Responsabilidad
+
+> [!CAUTION]
+> Esta biblioteca se proporciona solo con fines educativos y de investigación. Al usar esta biblioteca, aceptas cumplir con las leyes locales e internacionales de scraping de datos y privacidad. Los autores y contribuyentes no son responsables de ningún mal uso de este software. Respeta siempre los términos de servicio de los sitios web y los archivos robots.txt.
+
+## Licencia
+
+Este trabajo está licenciado bajo la Licencia BSD-3-Clause.
+
+## Agradecimientos
+
+Este proyecto incluye código adaptado de:
+- Parsel (Licencia BSD)—Usado para el submódulo [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py)
+
+---
+<div align="center"><small>Diseñado y elaborado con ❤️ por Karim Shoair.</small></div><br>

docs/README_JP.md

@@ -0,0 +1,426 @@
+<!-- mcp-name: io.github.D4Vinci/Scrapling -->
+
+<h1 align="center">
+    <a href="https://scrapling.readthedocs.io">
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_dark.svg?sanitize=true">
+          <img alt="Scrapling Poster" src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_light.svg?sanitize=true">
+        </picture>
+    </a>
+    <br>
+    <small>Effortless Web Scraping for the Modern Web</small>
+</h1>
+
+<p align="center">
+    <a href="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml" alt="Tests">
+        <img alt="Tests" src="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg"></a>
+    <a href="https://badge.fury.io/py/Scrapling" alt="PyPI version">
+        <img alt="PyPI version" src="https://badge.fury.io/py/Scrapling.svg"></a>
+    <a href="https://pepy.tech/project/scrapling" alt="PyPI Downloads">
+        <img alt="PyPI Downloads" src="https://static.pepy.tech/personalized-badge/scrapling?period=total&units=INTERNATIONAL_SYSTEM&left_color=GREY&right_color=GREEN&left_text=Downloads"></a>
+    <br/>
+    <a href="https://discord.gg/EMgGbDceNQ" alt="Discord" target="_blank">
+      <img alt="Discord" src="https://img.shields.io/discord/1360786381042880532?style=social&logo=discord&link=https%3A%2F%2Fdiscord.gg%2FEMgGbDceNQ">
+    </a>
+    <a href="https://x.com/Scrapling_dev" alt="X (formerly Twitter)">
+      <img alt="X (formerly Twitter) Follow" src="https://img.shields.io/twitter/follow/Scrapling_dev?style=social&logo=x&link=https%3A%2F%2Fx.com%2FScrapling_dev">
+    </a>
+    <br/>
+    <a href="https://pypi.org/project/scrapling/" alt="Supported Python versions">
+        <img alt="Supported Python versions" src="https://img.shields.io/pypi/pyversions/scrapling.svg"></a>
+</p>
+
+<p align="center">
+    <a href="https://scrapling.readthedocs.io/en/latest/parsing/selection/"><strong>選択メソッド</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/fetching/choosing/"><strong>Fetcherの選び方</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/architecture.html"><strong>スパイダー</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/proxy-blocking.html"><strong>プロキシローテーション</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/cli/overview/"><strong>CLI</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/ai/mcp-server/"><strong>MCPモード</strong></a>
+</p>
+
+Scraplingは、単一のリクエストから本格的なクロールまですべてを処理する適応型Web Scrapingフレームワークです。
+
+そのパーサーはウェブサイトの変更から学習し、ページが更新されたときに要素を自動的に再配置します。Fetcherはすぐに使えるCloudflare Turnstileなどのアンチボットシステムを回避します。そしてSpiderフレームワークにより、Pause & Resumeや自動Proxy回転機能を備えた並行マルチSessionクロールへとスケールアップできます — すべてわずか数行のPythonで。1つのライブラリ、妥協なし。
+
+リアルタイム統計とStreamingによる超高速クロール。Web Scraperによって、Web Scraperと一般ユーザーのために構築され、誰にでも何かがあります。
+
+```python
+from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
+StealthyFetcher.adaptive = True
+p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True)  # レーダーの下でウェブサイトを取得！
+products = p.css('.product', auto_save=True)                                        # ウェブサイトのデザイン変更に耐えるデータをスクレイプ！
+products = p.css('.product', adaptive=True)                                         # 後でウェブサイトの構造が変わったら、`adaptive=True`を渡して見つける！
+```
+または本格的なクロールへスケールアップ
+```python
+from scrapling.spiders import Spider, Response
+
+class MySpider(Spider):
+  name = "demo"
+  start_urls = ["https://example.com/"]
+
+  async def parse(self, response: Response):
+      for item in response.css('.product'):
+          yield {"title": item.css('h2::text').get()}
+
+MySpider().start()
+```
+
+
+# プラチナスポンサー
+
+<i><sub>ここに最初に表示される企業になりませんか？[こちら](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)をクリック</sub></i>
+# スポンサー
+
+<!-- sponsors -->
+
+<a href="https://www.scrapeless.com/en?utm_source=official&utm_term=scrapling" target="_blank" title="Effortless Web Scraping Toolkit for Business and Developers"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/scrapeless.jpg"></a>
+<a href="https://www.thordata.com/?ls=github&lk=github" target="_blank" title="Unblockable proxies and scraping infrastructure, delivering real-time, reliable web data to power AI models and workflows."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/thordata.jpg"></a>
+<a href="https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling" target="_blank" title="Evomi is your Swiss Quality Proxy Provider, starting at $0.49/GB"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/evomi.png"></a>
+<a href="https://serpapi.com/?utm_source=scrapling" target="_blank" title="Scrape Google and other search engines with SerpApi"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SerpApi.png"></a>
+<a href="https://visit.decodo.com/Dy6W0b" target="_blank" title="Try the Most Efficient Residential Proxies for Free"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/decodo.png"></a>
+<a href="https://petrosky.io/d4vinci" target="_blank" title="PetroSky delivers cutting-edge VPS hosting."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/petrosky.png"></a>
+<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
+<a href="https://proxyempire.io/" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
+<a href="https://hypersolutions.co/?utm_source=github&utm_medium=readme&utm_campaign=scrapling" target="_blank" title="Bot Protection Bypass API for Akamai, DataDome, Incapsula & Kasada"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/HyperSolutions.png"></a>
+
+
+<a href="https://www.swiftproxy.net/" target="_blank" title="Unlock Reliable Proxy Services with Swiftproxy!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/swiftproxy.png"></a>
+<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
+<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
+
+<!-- /sponsors -->
+
+<i><sub>ここに広告を表示したいですか？[こちら](https://github.com/sponsors/D4Vinci)をクリックして、あなたに合ったティアを選択してください！</sub></i>
+
+---
+
+## 主な機能
+
+### Spider — 本格的なクロールフレームワーク
+- 🕷️ **Scrapy風のSpider API**：`start_urls`、async `parse` callback、`Request`/`Response`オブジェクトでSpiderを定義。
+- ⚡ **並行クロール**：設定可能な並行数制限、ドメインごとのスロットリング、ダウンロード遅延。
+- 🔄 **マルチSessionサポート**：HTTPリクエストとステルスヘッドレスブラウザの統一インターフェース — IDによって異なるSessionにリクエストをルーティング。
+- 💾 **Pause & Resume**：Checkpointベースのクロール永続化。Ctrl+Cで正常にシャットダウン；再起動すると中断したところから再開。
+- 📡 **Streamingモード**：`async for item in spider.stream()`でリアルタイム統計とともにスクレイプされたアイテムをStreamingで受信 — UI、パイプライン、長時間実行クロールに最適。
+- 🛡️ **ブロックされたリクエストの検出**：カスタマイズ可能なロジックによるブロックされたリクエストの自動検出とリトライ。
+- 📦 **組み込みエクスポート**：フックや独自のパイプライン、または組み込みのJSON/JSONLで結果をエクスポート。それぞれ`result.items.to_json()` / `result.items.to_jsonl()`を使用。
+
+### Sessionサポート付き高度なウェブサイト取得
+- **HTTPリクエスト**：`Fetcher`クラスで高速かつステルスなHTTPリクエスト。ブラウザのTLS fingerprint、ヘッダーを模倣し、HTTP/3を使用可能。
+- **動的読み込み**：PlaywrightのChromiumとGoogle Chromeをサポートする`DynamicFetcher`クラスによる完全なブラウザ自動化で動的ウェブサイトを取得。
+- **アンチボット回避**：`StealthyFetcher`とfingerprint偽装による高度なステルス機能。自動化でCloudflareのTurnstile/Interstitialのすべてのタイプを簡単に回避。
+- **Session管理**：リクエスト間でCookieと状態を管理するための`FetcherSession`、`StealthySession`、`DynamicSession`クラスによる永続的なSessionサポート。
+- **Proxy回転**：すべてのSessionタイプに対応したラウンドロビンまたはカスタム戦略の組み込み`ProxyRotator`、さらにリクエストごとのProxyオーバーライド。
+- **ドメインブロック**：ブラウザベースのFetcherで特定のドメイン（およびそのサブドメイン）へのリクエストをブロック。
+- **asyncサポート**：すべてのFetcherおよび専用asyncSessionクラス全体での完全なasyncサポート。
+
+### 適応型スクレイピングとAI統合
+- 🔄 **スマート要素追跡**：インテリジェントな類似性アルゴリズムを使用してウェブサイトの変更後に要素を再配置。
+- 🎯 **スマート柔軟選択**：CSSセレクタ、XPathセレクタ、フィルタベース検索、テキスト検索、正規表現検索など。
+- 🔍 **類似要素の検出**：見つかった要素に類似した要素を自動的に特定。
+- 🤖 **AIと使用するMCPサーバー**：AI支援Web Scrapingとデータ抽出のための組み込みMCPサーバー。MCPサーバーは、AI（Claude/Cursorなど）に渡す前にScraplingを活用してターゲットコンテンツを抽出する強力でカスタムな機能を備えており、操作を高速化し、トークン使用量を最小限に抑えることでコストを削減します。（[デモ動画](https://www.youtube.com/watch?v=qyFk3ZNwOxE)）
+
+### 高性能で実戦テスト済みのアーキテクチャ
+- 🚀 **超高速**：ほとんどのPythonスクレイピングライブラリを上回る最適化されたパフォーマンス。
+- 🔋 **メモリ効率**：最小のメモリフットプリントのための最適化されたデータ構造と遅延読み込み。
+- ⚡ **高速JSONシリアル化**：標準ライブラリの10倍の速度。
+- 🏗️ **実戦テスト済み**：Scraplingは92%のテストカバレッジと完全な型ヒントカバレッジを備えているだけでなく、過去1年間に数百人のWeb Scraperによって毎日使用されてきました。
+
+### 開発者/Web Scraperにやさしい体験
+- 🎯 **インタラクティブWeb Scraping Shell**：Scrapling統合、ショートカット、curlリクエストをScraplingリクエストに変換したり、ブラウザでリクエスト結果を表示したりするなどの新しいツールを備えたオプションの組み込みIPython Shellで、Web Scrapingスクリプトの開発を加速。
+- 🚀 **ターミナルから直接使用**：オプションで、コードを一行も書かずにScraplingを使用してURLをスクレイプできます！
+- 🛠️ **豊富なナビゲーションAPI**：親、兄弟、子のナビゲーションメソッドによる高度なDOMトラバーサル。
+- 🧬 **強化されたテキスト処理**：組み込みの正規表現、クリーニングメソッド、最適化された文字列操作。
+- 📝 **自動セレクタ生成**：任意の要素に対して堅牢なCSS/XPathセレクタを生成。
+- 🔌 **馴染みのあるAPI**：Scrapy/Parselで使用されている同じ疑似要素を持つScrapy/BeautifulSoupに似た設計。
+- 📘 **完全な型カバレッジ**：優れたIDEサポートとコード補完のための完全な型ヒント。コードベース全体が変更のたびに**PyRight**と**MyPy**で自動的にスキャンされます。
+- 🔋 **すぐに使えるDockerイメージ**：各リリースで、すべてのブラウザを含むDockerイメージが自動的にビルドおよびプッシュされます。
+
+## はじめに
+
+深く掘り下げずに、Scraplingにできることの簡単な概要をお見せしましょう。
+
+### 基本的な使い方
+Sessionサポート付きHTTPリクエスト
+```python
+from scrapling.fetchers import Fetcher, FetcherSession
+
+with FetcherSession(impersonate='chrome') as session:  # ChromeのTLS fingerprintの最新バージョンを使用
+    page = session.get('https://quotes.toscrape.com/', stealthy_headers=True)
+    quotes = page.css('.quote .text::text').getall()
+
+# または一回限りのリクエストを使用
+page = Fetcher.get('https://quotes.toscrape.com/')
+quotes = page.css('.quote .text::text').getall()
+```
+高度なステルスモード
+```python
+from scrapling.fetchers import StealthyFetcher, StealthySession
+
+with StealthySession(headless=True, solve_cloudflare=True) as session:  # 完了するまでブラウザを開いたままにする
+    page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False)
+    data = page.css('#padded_content a').getall()
+
+# または一回限りのリクエストスタイル、このリクエストのためにブラウザを開き、完了後に閉じる
+page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare')
+data = page.css('#padded_content a').getall()
+```
+完全なブラウザ自動化
+```python
+from scrapling.fetchers import DynamicFetcher, DynamicSession
+
+with DynamicSession(headless=True, disable_resources=False, network_idle=True) as session:  # 完了するまでブラウザを開いたままにする
+    page = session.fetch('https://quotes.toscrape.com/', load_dom=False)
+    data = page.xpath('//span[@class="text"]/text()').getall()  # お好みであればXPathセレクタを使用
+
+# または一回限りのリクエストスタイル、このリクエストのためにブラウザを開き、完了後に閉じる
+page = DynamicFetcher.fetch('https://quotes.toscrape.com/')
+data = page.css('.quote .text::text').getall()
+```
+
+### Spider
+並行リクエスト、複数のSessionタイプ、Pause & Resumeを備えた本格的なクローラーを構築：
+```python
+from scrapling.spiders import Spider, Request, Response
+
+class QuotesSpider(Spider):
+    name = "quotes"
+    start_urls = ["https://quotes.toscrape.com/"]
+    concurrent_requests = 10
+
+    async def parse(self, response: Response):
+        for quote in response.css('.quote'):
+            yield {
+                "text": quote.css('.text::text').get(),
+                "author": quote.css('.author::text').get(),
+            }
+
+        next_page = response.css('.next a')
+        if next_page:
+            yield response.follow(next_page[0].attrib['href'])
+
+result = QuotesSpider().start()
+print(f"{len(result.items)}件の引用をスクレイプしました")
+result.items.to_json("quotes.json")
+```
+単一のSpiderで複数のSessionタイプを使用：
+```python
+from scrapling.spiders import Spider, Request, Response
+from scrapling.fetchers import FetcherSession, AsyncStealthySession
+
+class MultiSessionSpider(Spider):
+    name = "multi"
+    start_urls = ["https://example.com/"]
+
+    def configure_sessions(self, manager):
+        manager.add("fast", FetcherSession(impersonate="chrome"))
+        manager.add("stealth", AsyncStealthySession(headless=True), lazy=True)
+
+    async def parse(self, response: Response):
+        for link in response.css('a::attr(href)').getall():
+            # 保護されたページはステルスSessionを通してルーティング
+            if "protected" in link:
+                yield Request(link, sid="stealth")
+            else:
+                yield Request(link, sid="fast", callback=self.parse)  # 明示的なcallback
+```
+Checkpointを使用して長時間のクロールをPause & Resume：
+```python
+QuotesSpider(crawldir="./crawl_data").start()
+```
+Ctrl+Cを押すと正常に一時停止し、進捗は自動的に保存されます。後でSpiderを再度起動する際に同じ`crawldir`を渡すと、中断したところから再開します。
+
+### 高度なパースとナビゲーション
+```python
+from scrapling.fetchers import Fetcher
+
+# 豊富な要素選択とナビゲーション
+page = Fetcher.get('https://quotes.toscrape.com/')
+
+# 複数の選択メソッドで引用を取得
+quotes = page.css('.quote')  # CSSセレクタ
+quotes = page.xpath('//div[@class="quote"]')  # XPath
+quotes = page.find_all('div', {'class': 'quote'})  # BeautifulSoupスタイル
+# 以下と同じ
+quotes = page.find_all('div', class_='quote')
+quotes = page.find_all(['div'], class_='quote')
+quotes = page.find_all(class_='quote')  # など...
+# テキスト内容で要素を検索
+quotes = page.find_by_text('quote', tag='div')
+
+# 高度なナビゲーション
+quote_text = page.css('.quote')[0].css('.text::text').get()
+quote_text = page.css('.quote').css('.text::text').getall()  # チェーンセレクタ
+first_quote = page.css('.quote')[0]
+author = first_quote.next_sibling.css('.author::text')
+parent_container = first_quote.parent
+
+# 要素の関連性と類似性
+similar_elements = first_quote.find_similar()
+below_elements = first_quote.below_elements()
+```
+ウェブサイトを取得せずにパーサーをすぐに使用することもできます：
+```python
+from scrapling.parser import Selector
+
+page = Selector("<html>...</html>")
+```
+まったく同じ方法で動作します！
+
+### 非同期Session管理の例
+```python
+import asyncio
+from scrapling.fetchers import FetcherSession, AsyncStealthySession, AsyncDynamicSession
+
+async with FetcherSession(http3=True) as session:  # `FetcherSession`はコンテキストアウェアで、同期/非同期両方のパターンで動作可能
+    page1 = session.get('https://quotes.toscrape.com/')
+    page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135')
+
+# 非同期Sessionの使用
+async with AsyncStealthySession(max_pages=2) as session:
+    tasks = []
+    urls = ['https://example.com/page1', 'https://example.com/page2']
+
+    for url in urls:
+        task = session.fetch(url)
+        tasks.append(task)
+
+    print(session.get_pool_stats())  # オプション - ブラウザタブプールのステータス（ビジー/フリー/エラー）
+    results = await asyncio.gather(*tasks)
+    print(session.get_pool_stats())
+```
+
+## CLIとインタラクティブShell
+
+Scraplingには強力なコマンドラインインターフェー��が含まれています：
+
+[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339)
+
+インタラクティブWeb Scraping Shellを起動
+```bash
+scrapling shell
+```
+プログラミングせずに直接ページをファイルに抽出（デフォルトで`body`タグ内のコンテンツを抽出）。出力ファイルが`.txt`で終わる場合、ターゲットのテキストコンテンツが抽出されます。`.md`で終わる場合、HTMLコンテンツのMarkdown表現になります。`.html`で終わる場合、HTMLコンテンツそのものになります。
+```bash
+scrapling extract get 'https://example.com' content.md
+scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome'  # CSSセレクタ'#fromSkipToProducts'に一致するすべての要素
+scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless
+scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare
+```
+
+> [!NOTE]
+> MCPサーバーやインタラクティブWeb Scraping Shellなど、他にも多くの追加機能がありますが、このページは簡潔に保ちたいと思います。完全なドキュメントは[こちら](https://scrapling.readthedocs.io/en/latest/)をご覧ください
+
+## パフォーマンスベンチマーク
+
+Scraplingは強力であるだけでなく、超高速です。以下のベンチマークは、Scraplingのパーサーを他の人気ライブラリの最新バージョンと比較しています。
+
+### テキスト抽出速度テスト（5000個のネストされた要素）
+
+| # |      ライブラリ      | 時間(ms) | vs Scrapling |
+|---|:-----------------:|:---------:|:------------:|
+| 1 |     Scrapling     |   2.02    |     1.0x     |
+| 2 |   Parsel/Scrapy   |   2.04    |     1.01     |
+| 3 |     Raw Lxml      |   2.54    |    1.257     |
+| 4 |      PyQuery      |   24.17   |     ~12x     |
+| 5 |    Selectolax     |   82.63   |     ~41x     |
+| 6 |  MechanicalSoup   |  1549.71  |   ~767.1x    |
+| 7 |   BS4 with Lxml   |  1584.31  |   ~784.3x    |
+| 8 | BS4 with html5lib |  3391.91  |   ~1679.1x   |
+
+
+### 要素類似性とテキスト検索のパフォーマンス
+
+Scraplingの適応型要素検索機能は代替手段を大幅に上回ります：
+
+| ライブラリ     | 時間(ms) | vs Scrapling |
+|-------------|:---------:|:------------:|
+| Scrapling   |   2.39    |     1.0x     |
+| AutoScraper |   12.45   |    5.209x    |
+
+
+> すべてのベンチマークは100回以上の実行の平均を表します。方法論については[benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py)を参照してください。
+
+## インストール
+
+ScraplingにはPython 3.10以上が必要です：
+
+```bash
+pip install scrapling
+```
+
+このインストールにはパーサーエンジンとその依存関係のみが含まれており、Fetcherやコマンドライン依存関係は含まれていません。
+
+### オプションの依存関係
+
+1. 以下の追加機能、Fetcher、またはそれらのクラスのいずれかを使用する場合は、Fetcherの依存関係とブラウザの依存関係を次のようにインストールする必要があります：
+    ```bash
+    pip install "scrapling[fetchers]"
+
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
+    ```
+
+    これにより、すべてのブラウザ、およびそれらのシステム依存関係とfingerprint操作依存関係がダウンロードされます。
+
+    または、コマンドを実行する代わりにコードからインストールすることもできます：
+    ```python
+    from scrapling.cli import install
+
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
+2. 追加機能：
+   - MCPサーバー機能をインストール：
+       ```bash
+       pip install "scrapling[ai]"
+       ```
+   - Shell機能（Web Scraping Shellと`extract`コマンド）をインストール：
+       ```bash
+       pip install "scrapling[shell]"
+       ```
+   - すべてをインストール：
+       ```bash
+       pip install "scrapling[all]"
+       ```
+   これらの追加機能のいずれかの後（まだインストールしていない場合）、`scrapling install`でブラウザの依存関係をインストールする必要があることを忘れないでください
+
+### Docker
+DockerHubから次のコマンドですべての追加機能とブラウザを含むDockerイメージをインストールすることもできます：
+```bash
+docker pull pyd4vinci/scrapling
+```
+またはGitHubレジストリからダウンロード：
+```bash
+docker pull ghcr.io/d4vinci/scrapling:latest
+```
+このイメージは、GitHub Actionsとリポジトリの��インブランチを使用して自動的にビルドおよびプッシュされます。
+
+## 貢献
+
+貢献を歓迎します！始める前に[貢献ガイドライン](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md)をお読みください。
+
+## 免責事項
+
+> [!CAUTION]
+> このライブラリは教育および研究目的のみで提供されています。このライブラリを使用することにより、地域および国際的なデータスクレイピングおよびプライバシー法に準拠することに同意したものとみなされます。著者および貢献者は、このソフトウェアの誤用について責任を負いません。常にウェブサイトの利用規約とrobots.txtファイルを尊重してください。
+
+## ライセンス
+
+この作品はBSD-3-Clauseライセンスの下でライセンスされています。
+
+## 謝辞
+
+このプロジェクトには次から適応されたコードが含まれています：
+- Parsel（BSDライセンス）— [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py)サブモジュールに使用
+
+---
+<div align="center"><small>Karim Shoairによって❤️でデザインおよび作成されました。</small></div><br>

docs/README_RU.md

@@ -0,0 +1,426 @@
+<!-- mcp-name: io.github.D4Vinci/Scrapling -->
+
+<h1 align="center">
+    <a href="https://scrapling.readthedocs.io">
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_dark.svg?sanitize=true">
+          <img alt="Scrapling Poster" src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/cover_light.svg?sanitize=true">
+        </picture>
+    </a>
+    <br>
+    <small>Effortless Web Scraping for the Modern Web</small>
+</h1>
+
+<p align="center">
+    <a href="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml" alt="Tests">
+        <img alt="Tests" src="https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg"></a>
+    <a href="https://badge.fury.io/py/Scrapling" alt="PyPI version">
+        <img alt="PyPI version" src="https://badge.fury.io/py/Scrapling.svg"></a>
+    <a href="https://pepy.tech/project/scrapling" alt="PyPI Downloads">
+        <img alt="PyPI Downloads" src="https://static.pepy.tech/personalized-badge/scrapling?period=total&units=INTERNATIONAL_SYSTEM&left_color=GREY&right_color=GREEN&left_text=Downloads"></a>
+    <br/>
+    <a href="https://discord.gg/EMgGbDceNQ" alt="Discord" target="_blank">
+      <img alt="Discord" src="https://img.shields.io/discord/1360786381042880532?style=social&logo=discord&link=https%3A%2F%2Fdiscord.gg%2FEMgGbDceNQ">
+    </a>
+    <a href="https://x.com/Scrapling_dev" alt="X (formerly Twitter)">
+      <img alt="X (formerly Twitter) Follow" src="https://img.shields.io/twitter/follow/Scrapling_dev?style=social&logo=x&link=https%3A%2F%2Fx.com%2FScrapling_dev">
+    </a>
+    <br/>
+    <a href="https://pypi.org/project/scrapling/" alt="Supported Python versions">
+        <img alt="Supported Python versions" src="https://img.shields.io/pypi/pyversions/scrapling.svg"></a>
+</p>
+
+<p align="center">
+    <a href="https://scrapling.readthedocs.io/en/latest/parsing/selection/"><strong>Методы выбора</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/fetching/choosing/"><strong>Выбор Fetcher</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/architecture.html"><strong>Пауки</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/spiders/proxy-blocking.html"><strong>Ротация прокси</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/cli/overview/"><strong>CLI</strong></a>
+    &middot;
+    <a href="https://scrapling.readthedocs.io/en/latest/ai/mcp-server/"><strong>Режим MCP</strong></a>
+</p>
+
+Scrapling — это адаптивный фреймворк для Web Scraping, который берёт на себя всё: от одного запроса до полномасштабного обхода сайтов.
+
+Его парсер учится на изменениях сайтов и автоматически перемещает ваши элементы при обновлении страниц. Его Fetcher'ы обходят анти-бот системы вроде Cloudflare Turnstile прямо из коробки. А его Spider-фреймворк позволяет масштабироваться до параллельных, многосессионных обходов с Pause & Resume и автоматической ротацией Proxy — и всё это в нескольких строках Python. Одна библиотека, без компромиссов.
+
+Молниеносно быстрые обходы с отслеживанием статистики в реальном времени и Streaming. Создано веб-скраперами для веб-скраперов и обычных пользователей — здесь есть что-то для каждого.
+
+```python
+from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
+StealthyFetcher.adaptive = True
+p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True)  # Загрузите сайт незаметно!
+products = p.css('.product', auto_save=True)                                        # Скрапьте данные, которые переживут изменения дизайна сайта!
+products = p.css('.product', adaptive=True)                                         # Позже, если структура сайта изменится, передайте `adaptive=True`, чтобы найти их!
+```
+Или масштабируйте до полного обхода
+```python
+from scrapling.spiders import Spider, Response
+
+class MySpider(Spider):
+  name = "demo"
+  start_urls = ["https://example.com/"]
+
+  async def parse(self, response: Response):
+      for item in response.css('.product'):
+          yield {"title": item.css('h2::text').get()}
+
+MySpider().start()
+```
+
+
+# Платиновые спонсоры
+
+<i><sub>Хотите стать первой компанией, которая появится здесь? Нажмите [здесь](https://github.com/sponsors/D4Vinci/sponsorships?tier_id=586646)</sub></i>
+# Спонсоры
+
+<!-- sponsors -->
+
+<a href="https://www.scrapeless.com/en?utm_source=official&utm_term=scrapling" target="_blank" title="Effortless Web Scraping Toolkit for Business and Developers"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/scrapeless.jpg"></a>
+<a href="https://www.thordata.com/?ls=github&lk=github" target="_blank" title="Unblockable proxies and scraping infrastructure, delivering real-time, reliable web data to power AI models and workflows."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/thordata.jpg"></a>
+<a href="https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling" target="_blank" title="Evomi is your Swiss Quality Proxy Provider, starting at $0.49/GB"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/evomi.png"></a>
+<a href="https://serpapi.com/?utm_source=scrapling" target="_blank" title="Scrape Google and other search engines with SerpApi"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SerpApi.png"></a>
+<a href="https://visit.decodo.com/Dy6W0b" target="_blank" title="Try the Most Efficient Residential Proxies for Free"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/decodo.png"></a>
+<a href="https://petrosky.io/d4vinci" target="_blank" title="PetroSky delivers cutting-edge VPS hosting."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/petrosky.png"></a>
+<a href="https://hasdata.com/?utm_source=github&utm_medium=banner&utm_campaign=D4Vinci" target="_blank" title="The web scraping service that actually beats anti-bot systems!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/hasdata.png"></a>
+<a href="https://proxyempire.io/" target="_blank" title="Collect The Data Your Project Needs with the Best Residential Proxies"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png"></a>
+<a href="https://hypersolutions.co/?utm_source=github&utm_medium=readme&utm_campaign=scrapling" target="_blank" title="Bot Protection Bypass API for Akamai, DataDome, Incapsula & Kasada"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/HyperSolutions.png"></a>
+
+
+<a href="https://www.swiftproxy.net/" target="_blank" title="Unlock Reliable Proxy Services with Swiftproxy!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/swiftproxy.png"></a>
+<a href="https://www.rapidproxy.io/?ref=d4v" target="_blank" title="Affordable Access to the Proxy World – bypass CAPTCHAs blocks, and avoid additional costs."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/rapidproxy.jpg"></a>
+<a href="https://browser.cash/?utm_source=D4Vinci&utm_medium=referral" target="_blank" title="Browser Automation & AI Browser Agent Platform"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/browserCash.png"></a>
+
+<!-- /sponsors -->
+
+<i><sub>Хотите показать здесь свою рекламу? Нажмите [здесь](https://github.com/sponsors/D4Vinci) и выберите подходящий вам уровень!</sub></i>
+
+---
+
+## Ключевые особенности
+
+### Spider'ы — полноценный фреймворк для обхода сайтов
+- 🕷️ **Scrapy-подобный Spider API**: Определяйте Spider'ов с `start_urls`, async `parse` callback'ами и объектами `Request`/`Response`.
+- ⚡ **Параллельный обход**: Настраиваемые лимиты параллелизма, ограничение скорости по домену и задержки загрузки.
+- 🔄 **Поддержка нескольких сессий**: Единый интерфейс для HTTP-запросов и скрытных headless-браузеров в одном Spider — маршрутизируйте запросы к разным сессиям по ID.
+- 💾 **Pause & Resume**: Persistence обхода на основе Checkpoint'ов. Нажмите Ctrl+C для мягкой остановки; перезапустите, чтобы продолжить с того места, где вы остановились.
+- 📡 **Режим Streaming**: Стримьте извлечённые элементы по мере их поступления через `async for item in spider.stream()` со статистикой в реальном времени — идеально для UI, конвейеров и длительных обходов.
+- 🛡️ **Обнаружение заблокированных запросов**: Автоматическое обнаружение и повторная отправка заблокированных запросов с настраиваемо�� логикой.
+- 📦 **Встроенный экспорт**: Экспортируйте результаты через хуки и собственный конвейер или встроенный JSON/JSONL с `result.items.to_json()` / `result.items.to_jsonl()` соответственно.
+
+### Продвинутая загрузка сайтов с поддержкой Session
+- **HTTP-запросы**: Быстрые и скрытные HTTP-запросы с классом `Fetcher`. Может имитировать TLS fingerprint браузера, заголовки и использовать HTTP/3.
+- **Динамическая загрузка**: Загрузка динамических сайтов с полной автоматизацией браузера через класс `DynamicFetcher`, поддерживающий Chromium от Playwright и Google Chrome.
+- **Обход анти-ботов**: Расширенные возможности скрытности с `StealthyFetcher` и подмену fingerprint'ов. Может легко обойти все типы Cloudflare Turnstile/Interstitial с помощью автоматизации.
+- **Управление сессиями**: Поддержка постоянных сессий с классами `FetcherSession`, `StealthySession` и `DynamicSession` для управления cookie и состоянием между запросами.
+- **Ротация Proxy**: Встроенный `ProxyRotator` с циклической или пользовательскими стратегиями для всех типов сессий, а также переопределение Proxy для каждого запроса.
+- **Блокировка доменов**: Блокируйте запросы к определённым доменам (и их поддоменам) в браузерных Fetcher'ах.
+- **Поддержка async**: Полная async-поддержка во всех Fetcher'ах и выделенных async-классах сессий.
+
+### Адаптивный скрапинг и интеграция с ИИ
+- 🔄 **Умное отслеживание элементов**: Перемещайте элементы после изменений сайта с помощью интеллектуальных алгоритмов подобия.
+- 🎯 **Умный гибкий выбор**: CSS-селекторы, XPath-селекторы, поиск на основе фильтров, текстовый поиск, поиск по регулярным выражениям и многое другое.
+- 🔍 **Поиск похожих элементов**: Автоматически находите элементы, похожие на найденные.
+- 🤖 **MCP-сервер для использования с ИИ**: Встроенный MCP-сервер для Web Scraping с помощью ИИ и извлечения данных. MCP-сервер обладает мощными пользовательскими возможностями, которые используют Scrapling для извлечения целевого контента перед передачей его ИИ (Claude/Cursor/и т.д.), тем самым ускоряя операции и снижая затраты за счёт минимизации использования токенов. ([демо-видео](https://www.youtube.com/watch?v=qyFk3ZNwOxE))
+
+### Высокопроизводительная и проверенная в боях архитектура
+- 🚀 **Молниеносная скорость**: Оптимизированная производительность, превосходящая большинство Python-библиотек для скрапинга.
+- 🔋 **Эффективное использование памяти**: Оптимизированные структуры данных и ленивая загрузка для минимального потребления памяти.
+- ⚡ **Быстрая сериализация JSON**: В 10 раз быстрее стандартной библиотеки.
+- 🏗️ **Проверено в боях**: Scrapling имеет не только 92% покрытия тестами и полное покрытие type hints, но и ежедневно использовался сотнями веб-скраперов в течение последнего года.
+
+### Удобный для разработчиков/веб-скраперов опыт
+- 🎯 **Интерактивная Web Scraping Shell**: Опциональная встроенная IPython-оболочка с интеграцией Scrapling, ярлыками и новыми инструментами для ускорения разработки скриптов Web Scraping, такими как преобразование curl-запросов в запросы Scrapling и просмотр результатов з��просов в браузере.
+- 🚀 **Используйте прямо из терминала**: При желании вы можете использовать Scrapling для скрапинга URL без написания ни одной строки кода!
+- 🛠️ **Богатый API навигации**: Расширенный обход DOM с методами навигации по родителям, братьям и детям.
+- 🧬 **Улучшенная обработка текста**: Встроенные регулярные выражения, методы очистки и оптимизированные операции со строками.
+- 📝 **Автоматическая генерация селекторов**: Генерация надёжных CSS/XPath-селекторов для любого элемента.
+- 🔌 **Знакомый API**: Похож на Scrapy/BeautifulSoup с теми же псевдоэлементами, используемыми в Scrapy/Parsel.
+- 📘 **Полное покрытие типами**: Полные type hints для отличной поддержки IDE и автодополнения кода. Вся кодовая база автоматически проверяется **PyRight** и **MyPy** при каждом изменении.
+- 🔋 **Готовый Docker-образ**: С каждым релизом автоматически создаётся и публикуется Docker-образ, содержащий все браузеры.
+
+## Начало работы
+
+Давайте кратко покажем, на что способен Scrapling, без глубокого погружения.
+
+### Базовое использование
+HTTP-запросы с поддержкой Session
+```python
+from scrapling.fetchers import Fetcher, FetcherSession
+
+with FetcherSession(impersonate='chrome') as session:  # Используйте последнюю версию TLS fingerprint Chrome
+    page = session.get('https://quotes.toscrape.com/', stealthy_headers=True)
+    quotes = page.css('.quote .text::text').getall()
+
+# Или используйте одноразовые запросы
+page = Fetcher.get('https://quotes.toscrape.com/')
+quotes = page.css('.quote .text::text').getall()
+```
+Расширенный режим скрытности
+```python
+from scrapling.fetchers import StealthyFetcher, StealthySession
+
+with StealthySession(headless=True, solve_cloudflare=True) as session:  # Держите браузер открытым, пока не закончите
+    page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False)
+    data = page.css('#padded_content a').getall()
+
+# Или используйте стиль одноразового запроса — открывает браузер для этого запроса, затем закрывает его после завершения
+page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare')
+data = page.css('#padded_content a').getall()
+```
+Полная автоматизация браузера
+```python
+from scrapling.fetchers import DynamicFetcher, DynamicSession
+
+with DynamicSession(headless=True, disable_resources=False, network_idle=True) as session:  # Держите браузер открытым, пока не закончите
+    page = session.fetch('https://quotes.toscrape.com/', load_dom=False)
+    data = page.xpath('//span[@class="text"]/text()').getall()  # XPath-селектор, если вы предпочитаете его
+
+# Или используйте стиль одноразового запроса — открывает браузер для этого запроса, затем закрывает его после завершения
+page = DynamicFetcher.fetch('https://quotes.toscrape.com/')
+data = page.css('.quote .text::text').getall()
+```
+
+### Spider'ы
+Создавайте полноценные обходчики с параллельными запросами, несколькими типами сессий и Pause & Resume:
+```python
+from scrapling.spiders import Spider, Request, Response
+
+class QuotesSpider(Spider):
+    name = "quotes"
+    start_urls = ["https://quotes.toscrape.com/"]
+    concurrent_requests = 10
+
+    async def parse(self, response: Response):
+        for quote in response.css('.quote'):
+            yield {
+                "text": quote.css('.text::text').get(),
+                "author": quote.css('.author::text').get(),
+            }
+
+        next_page = response.css('.next a')
+        if next_page:
+            yield response.follow(next_page[0].attrib['href'])
+
+result = QuotesSpider().start()
+print(f"Извлечено {len(result.items)} цитат")
+result.items.to_json("quotes.json")
+```
+Используйте несколько типов сессий в одном Spider:
+```python
+from scrapling.spiders import Spider, Request, Response
+from scrapling.fetchers import FetcherSession, AsyncStealthySession
+
+class MultiSessionSpider(Spider):
+    name = "multi"
+    start_urls = ["https://example.com/"]
+
+    def configure_sessions(self, manager):
+        manager.add("fast", FetcherSession(impersonate="chrome"))
+        manager.add("stealth", AsyncStealthySession(headless=True), lazy=True)
+
+    async def parse(self, response: Response):
+        for link in response.css('a::attr(href)').getall():
+            # Направляйте защищённые страницы через stealth-сессию
+            if "protected" in link:
+                yield Request(link, sid="stealth")
+            else:
+                yield Request(link, sid="fast", callback=self.parse)  # явный callback
+```
+Приостанавливайте и возобновляйте длительные обходы с помощью Checkpoint'ов, запуская Spider следующим образом:
+```python
+QuotesSpider(crawldir="./crawl_data").start()
+```
+Нажмите Ctrl+C для мягкой остановки — прогресс сохраняется автоматически. Позже, когда вы снова запустите Spider, передайте тот же `crawldir`, и он продолжит с того места, где остановился.
+
+### Продвинутый парсинг и навигация
+```python
+from scrapling.fetchers import Fetcher
+
+# Богатый выбор элементов и навигация
+page = Fetcher.get('https://quotes.toscrape.com/')
+
+# Получение цитат различными методами выбора
+quotes = page.css('.quote')  # CSS-селектор
+quotes = page.xpath('//div[@class="quote"]')  # XPath
+quotes = page.find_all('div', {'class': 'quote'})  # В стиле BeautifulSoup
+# То же самое, что
+quotes = page.find_all('div', class_='quote')
+quotes = page.find_all(['div'], class_='quote')
+quotes = page.find_all(class_='quote')  # и так далее...
+# Найти элемент по текстовому содержимому
+quotes = page.find_by_text('quote', tag='div')
+
+# Продвинутая навигация
+quote_text = page.css('.quote')[0].css('.text::text').get()
+quote_text = page.css('.quote').css('.text::text').getall()  # Цепочка селекторов
+first_quote = page.css('.quote')[0]
+author = first_quote.next_sibling.css('.author::text')
+parent_container = first_quote.parent
+
+# Связи элементов и подобие
+similar_elements = first_quote.find_similar()
+below_elements = first_quote.below_elements()
+```
+Вы можете использовать парсер напрямую, если не хотите загружать сайты, как показано ниже:
+```python
+from scrapling.parser import Selector
+
+page = Selector("<html>...</html>")
+```
+И он работает точно так же!
+
+### Примеры async Session
+```python
+import asyncio
+from scrapling.fetchers import FetcherSession, AsyncStealthySession, AsyncDynamicSession
+
+async with FetcherSession(http3=True) as session:  # `FetcherSession` контекстно-осведомлён и может работать как в sync, так и в async-режимах
+    page1 = session.get('https://quotes.toscrape.com/')
+    page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135')
+
+# Использование async-сессии
+async with AsyncStealthySession(max_pages=2) as session:
+    tasks = []
+    urls = ['https://example.com/page1', 'https://example.com/page2']
+
+    for url in urls:
+        task = session.fetch(url)
+        tasks.append(task)
+
+    print(session.get_pool_stats())  # Опционально — статус пула вкладок браузера (занят/свободен/ошибка)
+    results = await asyncio.gather(*tasks)
+    print(session.get_pool_stats())
+```
+
+## CLI и интерактивная Shell
+
+Scrapling включает мощный интерфейс командной строки:
+
+[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339)
+
+Запустить интерактивную Web Scraping Shell
+```bash
+scrapling shell
+```
+Извлечь страницы в файл напрямую без программирования (по умолчанию извлекает содержимое внутри тега `body`). Если выходной файл заканчивается на `.txt`, будет извлечено текстовое содержимое цели. Если заканчивается на `.md`, это будет Markdown-представление HTML-содержимого; если заканчивается на `.html`, это будет само HTML-содержимое.
+```bash
+scrapling extract get 'https://example.com' content.md
+scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome'  # Все элементы, соответствующие CSS-селектору '#fromSkipToProducts'
+scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless
+scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare
+```
+
+> [!NOTE]
+> Есть множество дополнительных возможностей, но мы хотим сохранить эту страницу краткой, включая MCP-сервер и интерактивную Web Scraping Shell. Ознакомьтесь с полной документацией [здесь](https://scrapling.readthedocs.io/en/latest/)
+
+## Тесты производительности
+
+Scrapling не только мощный — он ещё и невероятно быстрый. Следующие тесты производительности сравнивают парсер Scrapling с последними версиями других популярных библиотек.
+
+### Тест скорости извлечения текста (5000 вложенных элементов)
+
+| # |    Библиотека     | Время (мс) | vs Scrapling |
+|---|:-----------------:|:----------:|:------------:|
+| 1 |     Scrapling     |    2.02    |     1.0x     |
+| 2 |   Parsel/Scrapy   |    2.04    |     1.01     |
+| 3 |     Raw Lxml      |    2.54    |    1.257     |
+| 4 |      PyQuery      |   24.17    |     ~12x     |
+| 5 |    Selectolax     |   82.63    |     ~41x     |
+| 6 |  MechanicalSoup   |  1549.71   |   ~767.1x    |
+| 7 |   BS4 with Lxml   |  1584.31   |   ~784.3x    |
+| 8 | BS4 with html5lib |  3391.91   |   ~1679.1x   |
+
+
+### Производительность подобия элементов и текстового поиска
+
+Возможности адаптивного поиска элементов Scrapling значительно превосходят альтернативы:
+
+| Библиотека  | Время (мс) | vs Scrapling |
+|-------------|:----------:|:------------:|
+| Scrapling   |    2.39    |     1.0x     |
+| AutoScraper |   12.45    |    5.209x    |
+
+
+> Все тесты производительности представляют собой средние значения более 100 запусков. См. [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) для методологии.
+
+## Установка
+
+Scrapling требует Python 3.10 или выше:
+
+```bash
+pip install scrapling
+```
+
+Эта установка включает только движок парсера и его зависимости, без каких-либо Fetcher'ов или зависимостей командной строки.
+
+### Опциональные зависимости
+
+1. Если вы собираетесь использовать какие-либо из дополнительных возможностей ниже, Fetcher'ы или их классы, вам необходимо установить зависимости Fetcher'ов и браузеров следующим образом:
+    ```bash
+    pip install "scrapling[fetchers]"
+
+    scrapling install           # normal install
+    scrapling install  --force  # force reinstall
+    ```
+
+    Это загрузит все браузеры вместе с их системными зависимостями и зависимостями для манипуляции fingerprint'ами.
+
+    Или вы можете установить их из кода вместо выполнения команды:
+    ```python
+    from scrapling.cli import install
+
+    install([], standalone_mode=False)          # normal install
+    install(["--force"], standalone_mode=False) # force reinstall
+    ```
+
+2. Дополнительные возможности:
+   - Установить функцию MCP-сервера:
+       ```bash
+       pip install "scrapling[ai]"
+       ```
+   - Установить функции Shell (Web Scraping Shell и команда `extract`):
+       ```bash
+       pip install "scrapling[shell]"
+       ```
+   - Установить всё:
+       ```bash
+       pip install "scrapling[all]"
+       ```
+   Помните, что вам нужно установить зависимости браузеров с помощью `scrapling install` после любого из этих дополнений (если вы ещё этого не сделали)
+
+### Docker
+Вы также можете установить Docker-образ со всеми дополнениями и браузерами с помощью следующей команды из DockerHub:
+```bash
+docker pull pyd4vinci/scrapling
+```
+Или скачайте его из реестра GitHub:
+```bash
+docker pull ghcr.io/d4vinci/scrapling:latest
+```
+Этот образ автоматически создаётся и публикуе��ся с помощью GitHub Actions и основной ветки репозитория.
+
+## Участие в разработке
+
+Мы приветствуем участие! Пожалуйста, прочитайте наши [руководства по участию в разработке](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md) перед началом работы.
+
+## Отказ от ответственности
+
+> [!CAUTION]
+> Эта библиотека предоставляется только в образовательных и исследовательских целях. Используя эту библиотеку, вы соглашаетесь соблюдать местные и международные законы о скрапинге данных и конфиденциальности. Авторы и участники не несут ответственности за любое неправомерное использование этого программного обеспечения. Всегда уважайте условия обслуживания веб-сайтов и файлы robots.txt.
+
+## Лицензия
+
+Эта работа лицензирована по лицензии BSD-3-Clause.
+
+## Благодарности
+
+Этот проект включает код, адаптированный из:
+- Parsel (лицензия BSD) — Используется для подмодуля [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py)
+
+---
+<div align="center"><small>Разработано и создано с ❤️ Карим Шоаир.</small></div><br>

docs/ai/mcp-server.md

@@ -0,0 +1,294 @@
+# Scrapling MCP Server Guide
+
+<iframe width="560" height="315" src="https://www.youtube.com/embed/qyFk3ZNwOxE?si=3FHzgcYCb66iJ6e3" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
+
+The **Scrapling MCP Server** is a new feature that brings Scrapling's powerful Web Scraping capabilities directly to your favorite AI chatbot or AI agent. This integration allows you to scrape websites, extract data, and bypass anti-bot protections conversationally through Claude's AI interface or any interface that supports MCP.
+
+## Features
+
+The Scrapling MCP Server provides six powerful tools for web scraping:
+
+### 🚀 Basic HTTP Scraping
+- **`get`**: Fast HTTP requests with browser fingerprint impersonation, generating real browser headers matching the TLS version, HTTP/3, and more!
+- **`bulk_get`**: An async version of the above tool that allows scraping of multiple URLs at the same time!
+
+### 🌐 Dynamic Content Scraping  
+- **`fetch`**: Rapidly fetch dynamic content with Chromium/Chrome browser with complete control over the request/browser, and more!
+- **`bulk_fetch`**: An async version of the above tool that allows scraping of multiple URLs in different browser tabs at the same time!
+
+### 🔒 Stealth Scraping
+- **`stealthy_fetch`**: Uses our Stealthy browser to bypass Cloudflare Turnstile/Interstitial and other anti-bot systems with complete control over the request/browser! 
+- **`bulk_stealthy_fetch`**: An async version of the above tool that allows stealth scraping of multiple URLs in different browser tabs at the same time!
+
+### Key Capabilities
+- **Smart Content Extraction**: Convert web pages/elements to Markdown, HTML, or extract a clean version of the text content
+- **CSS Selector Support**: Use the Scrapling engine to target specific elements with precision before handing the content to the AI
+- **Anti-Bot Bypass**: Handle Cloudflare Turnstile, Interstitial, and other protections
+- **Proxy Support**: Use proxies for anonymity and geo-targeting
+- **Browser Impersonation**: Mimic real browsers with TLS fingerprinting, real browser headers matching that version, and more
+- **Parallel Processing**: Scrape multiple URLs concurrently for efficiency
+
+#### But why use Scrapling MCP Server instead of other available tools?
+
+Aside from its stealth capabilities and ability to bypass Cloudflare Turnstile/Interstitial, Scrapling's server is the only one that lets you select specific elements to pass to the AI, saving a lot of time and tokens!
+
+The way other servers work is that they extract the content, then pass it all to the AI to extract the fields you want. This causes the AI to consume far more tokens than needed (from irrelevant content). Scrapling solves this problem by allowing you to pass a CSS selector to narrow down the content you want before passing it to the AI, which makes the whole process much faster and more efficient.
+
+If you don't know how to write/use CSS selectors, don't worry. You can tell the AI in the prompt to write selectors to match possible fields for you and watch it try different combinations until it finds the right one, as we will show in the examples section.
+
+## Installation
+
+Install Scrapling with MCP Support, then double-check that the browser dependencies are installed.
+
+```bash
+# Install Scrapling with MCP server dependencies
+pip install "scrapling[ai]"
+
+# Install browser dependencies
+scrapling install
+```
+
+Or use the Docker image directly from the Docker registry:
+```bash
+docker pull pyd4vinci/scrapling
+```
+Or download it from the GitHub registry:
+```bash
+docker pull ghcr.io/d4vinci/scrapling:latest
+```
+
+## Setting up the MCP Server
+
+Here we will explain how to add Scrapling MCP Server to [Claude Desktop](https://claude.ai/download) and [Claude Code](https://www.anthropic.com/claude-code), but the same logic applies to any other chatbot that supports MCP:
+
+### Claude Desktop
+
+1. Open Claude Desktop
+2. Click the hamburger menu (☰) at the top left → Settings → Developer → Edit Config
+3. Add the Scrapling MCP server configuration:
+```json
+"ScraplingServer": {
+  "command": "scrapling",
+  "args": [
+    "mcp"
+  ]
+}
+```
+If that's the first MCP server you're adding, set the content of the file to this: 
+```json
+{
+  "mcpServers": {
+    "ScraplingServer": {
+      "command": "scrapling",
+      "args": [
+        "mcp"
+      ]
+    }
+  }
+}
+```
+As per the [official article](https://modelcontextprotocol.io/quickstart/user), this action either creates a new configuration file if none exists or opens your existing configuration. The file is located at
+
+1. **MacOS**: `~/Library/Application Support/Claude/claude_desktop_config.json`
+2. **Windows**: `%APPDATA%\Claude\claude_desktop_config.json`
+
+To ensure it's working, use the full path to the `scrapling` executable. Open the terminal and execute the following command:
+
+1. **MacOS**: `which scrapling`
+2. **Windows**: `where scrapling`
+
+For me, on my Mac, it returned `/Users/<MyUsername>/.venv/bin/scrapling`, so the config I used in the end is:
+```json
+{
+  "mcpServers": {
+    "ScraplingServer": {
+      "command": "/Users/<MyUsername>/.venv/bin/scrapling",
+      "args": [
+        "mcp"
+      ]
+    }
+  }
+}
+```
+#### Docker
+If you are using the Docker image, then it would be something like
+```json
+{
+  "mcpServers": {
+    "ScraplingServer": {
+      "command": "docker",
+      "args": [
+        "run", "-i", "--rm", "scrapling", "mcp"
+      ]
+    }
+  }
+}
+```
+
+The same logic applies to [Cursor](https://cursor.com/docs/context/mcp), [WindSurf](https://windsurf.com/university/tutorials/configuring-first-mcp-server), and others.
+
+### Claude Code
+Here it's much simpler to do. If you have [Claude Code](https://www.anthropic.com/claude-code) installed, open the terminal and execute the following command:
+
+```bash
+claude mcp add ScraplingServer "/Users/<MyUsername>/.venv/bin/scrapling" mcp
+```
+Same as above, to get Scrapling's executable path, open the terminal and execute the following command:
+
+1. **MacOS**: `which scrapling`
+2. **Windows**: `where scrapling`
+
+Here's the main article from Anthropic on [how to add MCP servers to Claude code](https://docs.anthropic.com/en/docs/claude-code/mcp#option-1%3A-add-a-local-stdio-server) for further details.
+
+
+Then, after you've added the server, you need to completely quit and restart the app you used above. In Claude Desktop, you should see an MCP server indicator (🔧) in the bottom-right corner of the chat input or see `ScraplingServer` in the `Search and tools` dropdown in the chat input box.
+
+### Streamable HTTP
+As per version 0.3.6, we have added the ability to make the MCP server use the 'Streamable HTTP' transport mode instead of the traditional 'stdio' transport.
+
+So instead of using the following command (the 'stdio' one):
+```bash
+scrapling mcp
+```
+Use the following to enable 'Streamable HTTP' transport mode:
+```bash
+scrapling mcp --http
+```
+Hence, the default value for the host the server is listening to is '0.0.0.0' and the port is 8000, which both can be configured as below:
+```bash
+scrapling mcp --http --host '127.0.0.1' --port 8000
+```
+
+## Examples
+
+Now we will show you some examples of prompts we used while testing the MCP server, but you are probably more creative than we are and better at prompt engineering than we are :)
+
+We will gradually go from simple prompts to more complex ones. We will use Claude Desktop for the examples, but the same logic applies to the rest, of course.
+
+1. **Basic Web Scraping**
+
+    Extract the main content from a webpage as Markdown:
+    
+    ```
+    Scrape the main content from https://example.com and convert it to markdown format.
+    ```
+    
+    Claude will use the `get` tool to fetch the page and return clean, readable content. If it fails, it will continue retrying every second for 3 attempts, unless you instruct it otherwise. If it fails to retrieve content for any reason, such as protection or if it's a dynamic website, it will automatically try the other tools. If Claude didn't do that automatically for some reason, you can add that to the prompt.
+    
+    A more optimized version of the same prompt would be:
+    ```
+    Use regular requests to scrape the main content from https://example.com and convert it to markdown format.
+    ```
+    This tells Claude which tool to use here, so it doesn't have to guess. Sometimes it will start using normal requests on its own, and at other times, it will assume browsers are better suited for this website without any apparent reason. As a rule of thumb, you should always tell Claude which tool to use to save time and money and get consistent results.
+
+2. **Targeted Data Extraction**
+
+    Extract specific elements using CSS selectors:
+    
+    ```
+    Get all product titles from https://shop.example.com using the CSS selector '.product-title'. If the request fails, retry up to 5 times every 10 seconds.
+    ```
+    
+    The server will extract only the elements matching your selector and return them as a structured list. Notice I told it to set the tool to try up to 5 times in case the website has connection issues, but the default setting should be fine for most cases.
+
+3. **E-commerce Data Collection**
+
+    Another example of a bit more complex prompt:
+    ```
+    Extract product information from these e-commerce URLs using bulk browser fetches:
+    - https://shop1.com/product-a
+    - https://shop2.com/product-b  
+    - https://shop3.com/product-c
+    
+    Get the product names, prices, and descriptions from each page.
+    ```
+    
+    Claude will use `bulk_fetch` to concurrently scrape all URLs, then analyze the extracted data.
+
+4. **More advanced workflow**
+
+    Let's say I want to get all the action games available on PlayStation's store first page right now. I can use the following prompt to do that:
+    ```
+    Extract the URLs of all games in this page, then do a bulk request to them and return a list of all action games: https://store.playstation.com/en-us/pages/browse
+    ```
+    Note that I instructed it to use a bulk request for all the URLs collected. If I hadn't mentioned it, sometimes it works as intended, and other times it makes a separate request to each URL, which takes significantly longer. This prompt takes approximately one minute to complete.
+    
+    However, because I wasn't specific enough, it actually used the `stealthy_fetch` here and the `bulk_stealthy_fetch` in the second step, which unnecessarily consumed a large number of tokens. A better prompt would be:
+    ```
+    Use normal requests to extract the URLs of all games in this page, then do a bulk request to them and return a list of all action games: https://store.playstation.com/en-us/pages/browse
+    ```
+    And if you know how to write CSS selectors, you can instruct Claude to apply the selectors to the elements you want, and it will nearly complete the task immediately.
+    ```
+    Use normal requests to extract the URLs of all games on the page below, then perform a bulk request to them and return a list of all action games.
+    The selector for games in the first page is `[href*="/concept/"]` and the selector for the genre in the second request is `[data-qa="gameInfo#releaseInformation#genre-value"]`.
+    
+    URL: https://store.playstation.com/en-us/pages/browse
+    ```
+
+5. **Get data from a website with Cloudflare protection**
+
+    If you think the website you are targeting has Cloudflare protection, tell Claude instead of letting it discover it on its own.
+    ```
+    What's the price of this product? Be cautious, as it utilizes Cloudflare's Turnstile protection. Make the browser visible while you work.
+
+    https://ao.com/product/oo101uk-ninja-woodfire-outdoor-pizza-oven-brown-99357-685.aspx
+    ```
+
+6. **Long workflow**
+
+    You can, for example, use a prompt like this:
+    ```
+    Extract all product URLs for the following category, then return the prices and details for the first 3 products.
+    
+    https://www.arnotts.ie/furniture/bedroom/bed-frames/
+    ```
+    But a better prompt would be:
+    ```
+    Go to the following category URL and extract all product URLs using the CSS selector "a". Then, fetch the first 3 product pages in parallel and extract each product’s price and details.
+    
+    Keep the output in markdown format to reduce irrelevant content.
+    
+    Category URL:
+    https://www.arnotts.ie/furniture/bedroom/bed-frames/
+    ```
+
+And so on, you get the idea. Your creativity is the key here.
+
+## Best Practices
+
+Here is some technical advice for you.
+
+### 1. Choose the Right Tool
+- **`get`**: Fast, simple websites
+- **`fetch`**: Sites with JavaScript/dynamic content  
+- **`stealthy_fetch`**: Protected sites, Cloudflare, anti-bot systems
+
+### 2. Optimize Performance
+- Use bulk tools for multiple URLs
+- Disable unnecessary resources
+- Set appropriate timeouts
+- Use CSS selectors for targeted extraction
+
+### 3. Handle Dynamic Content
+- Use `network_idle` for SPAs
+- Set `wait_selector` for specific elements
+- Increase timeout for slow-loading sites
+
+### 4. Data Quality
+- Use `main_content_only=true` to avoid navigation/ads
+- Choose an appropriate `extraction_type` for your use case
+
+## Legal and Ethical Considerations
+
+⚠️ **Important Guidelines:**
+
+- **Check robots.txt**: Visit `https://website.com/robots.txt` to see scraping rules
+- **Respect rate limits**: Don't overwhelm servers with requests
+- **Terms of Service**: Read and comply with website terms
+- **Copyright**: Respect intellectual property rights
+- **Privacy**: Be mindful of personal data protection laws
+- **Commercial use**: Ensure you have permission for business purposes
+
+---
+
+*Built with ❤️ by the Scrapling team. Happy scraping!*

docs/api-reference/custom-types.md

@@ -0,0 +1,26 @@
+---
+search:
+  exclude: true
+---
+
+# Custom Types API Reference
+
+Here's the reference information for all custom types of classes Scrapling implemented, with all their parameters, attributes, and methods.
+
+You can import all of them directly like below:
+
+```python
+from scrapling.core.custom_types import TextHandler, TextHandlers, AttributesHandler
+```
+
+## ::: scrapling.core.custom_types.TextHandler
+    handler: python
+    :docstring:
+
+## ::: scrapling.core.custom_types.TextHandlers
+    handler: python
+    :docstring:
+
+## ::: scrapling.core.custom_types.AttributesHandler
+    handler: python
+    :docstring:

docs/api-reference/fetchers.md

@@ -0,0 +1,63 @@
+---
+search:
+  exclude: true
+---
+
+# Fetchers Classes
+
+Here's the reference information for all fetcher-type classes' parameters, attributes, and methods.
+
+You can import all of them directly like below:
+
+```python
+from scrapling.fetchers import (
+    Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher,
+    FetcherSession, AsyncStealthySession, StealthySession, DynamicSession, AsyncDynamicSession
+)
+```
+
+## ::: scrapling.fetchers.Fetcher
+    handler: python
+    :docstring:
+
+## ::: scrapling.fetchers.AsyncFetcher
+    handler: python
+    :docstring:
+
+## ::: scrapling.fetchers.DynamicFetcher
+    handler: python
+    :docstring:
+
+## ::: scrapling.fetchers.StealthyFetcher
+    handler: python
+    :docstring:
+
+
+## Session Classes
+
+### HTTP Sessions
+
+## ::: scrapling.fetchers.FetcherSession
+    handler: python
+    :docstring:
+
+### Stealth Sessions
+
+## ::: scrapling.fetchers.StealthySession
+    handler: python
+    :docstring:
+
+## ::: scrapling.fetchers.AsyncStealthySession
+    handler: python
+    :docstring:
+
+### Dynamic Sessions
+
+## ::: scrapling.fetchers.DynamicSession
+    handler: python
+    :docstring:
+
+## ::: scrapling.fetchers.AsyncDynamicSession
+    handler: python
+    :docstring:
+

docs/api-reference/mcp-server.md

@@ -0,0 +1,39 @@
+---
+search:
+  exclude: true
+---
+
+# MCP Server API Reference
+
+The **Scrapling MCP Server** provides six powerful tools for web scraping through the Model Context Protocol (MCP). This server integrates Scrapling's capabilities directly into AI chatbots and agents, allowing conversational web scraping with advanced anti-bot bypass features.
+
+You can start the MCP server by running:
+
+```bash
+scrapling mcp
+```
+
+Or import the server class directly:
+
+```python
+from scrapling.core.ai import ScraplingMCPServer
+
+server = ScraplingMCPServer()
+server.serve(http=False, host="0.0.0.0", port=8000)
+```
+
+## Response Model
+
+The standardized response structure that's returned by all MCP server tools:
+
+## ::: scrapling.core.ai.ResponseModel
+    handler: python
+    :docstring:
+
+## MCP Server Class
+
+The main MCP server class that provides all web scraping tools:
+
+## ::: scrapling.core.ai.ScraplingMCPServer
+    handler: python
+    :docstring:

docs/api-reference/proxy-rotation.md

@@ -0,0 +1,18 @@
+---
+search:
+  exclude: true
+---
+
+# Proxy Rotation
+
+The `ProxyRotator` class provides thread-safe proxy rotation for any fetcher or session.
+
+You can import it directly like below:
+
+```python
+from scrapling.fetchers import ProxyRotator
+```
+
+## ::: scrapling.engines.toolbelt.proxy_rotation.ProxyRotator
+    handler: python
+    :docstring:

docs/api-reference/response.md

@@ -0,0 +1,18 @@
+---
+search:
+  exclude: true
+---
+
+# Response Class
+
+The `Response` class wraps HTTP responses returned by all fetchers, providing access to status, headers, body, cookies, and a `Selector` for parsing.
+
+You can import the `Response` class like below:
+
+```python
+from scrapling.engines.toolbelt.custom import Response
+```
+
+## ::: scrapling.engines.toolbelt.custom.Response
+    handler: python
+    :docstring:

docs/api-reference/selector.md

@@ -0,0 +1,25 @@
+---
+search:
+  exclude: true
+---
+
+# Selector Class
+
+The `Selector` class is the core parsing engine in Scrapling that provides HTML parsing and element selection capabilities.
+
+Here's the reference information for the `Selector` class, with all its parameters, attributes, and methods.
+
+You can import the `Selector` class directly from `scrapling`:
+
+```python
+from scrapling.parser import Selector
+```
+
+## ::: scrapling.parser.Selector
+    handler: python
+    :docstring:
+
+## ::: scrapling.parser.Selectors
+    handler: python
+    :docstring:
+

docs/api-reference/spiders.md

@@ -0,0 +1,42 @@
+---
+search:
+  exclude: true
+---
+
+# Spider Classes
+
+Here's the reference information for the spider framework classes' parameters, attributes, and methods.
+
+You can import them directly like below:
+
+```python
+from scrapling.spiders import Spider, Request, CrawlResult, SessionManager, Response
+```
+
+## ::: scrapling.spiders.Spider
+    handler: python
+    :docstring:
+
+## ::: scrapling.spiders.Request
+    handler: python
+    :docstring:
+
+## Result Classes
+
+## ::: scrapling.spiders.result.CrawlResult
+    handler: python
+    :docstring:
+
+## ::: scrapling.spiders.result.CrawlStats
+    handler: python
+    :docstring:
+
+## ::: scrapling.spiders.result.ItemList
+    handler: python
+    :docstring:
+
+## Session Management
+
+## ::: scrapling.spiders.session.SessionManager
+    handler: python
+    :docstring: