scripts/fetch_company_docs.py

"""
fetch_company_docs.py
=====================
Fetches public help-center / documentation pages from 5 major product companies
and saves them as clean .txt files under data/<company>/.

Companies (chosen for large user bases + strong support-bot value):
  1. Notion        – 30 M+ users  – workspace / notes
  2. Slack         – 20 M+ DAU   – team messaging
  3. GitHub        – 100 M+ devs  – code hosting / collaboration
  4. Zoom          – 300 M+ meetings/day – video conferencing
  5. Shopify       – 2 M+ merchants – e-commerce

Usage:
    python scripts/fetch_company_docs.py           # fetch all companies
    python scripts/fetch_company_docs.py --company notion slack

Future cron job:
    Add this script to cron (e.g. weekly) and pipe output into ingest:
        0 2 * * 0 cd /app && python scripts/fetch_company_docs.py && \
                             python main.py ingest data/notion data/slack ...
"""
from __future__ import annotations

import argparse
import re
import sys
import time
from pathlib import Path
from typing import NamedTuple

# ── Dependency check ──────────────────────────────────────────────────────────
try:
    import requests
    from bs4 import BeautifulSoup
except ImportError:
    sys.exit(
        "Missing dependencies. Run:\n"
        "  pip install requests beautifulsoup4 lxml"
    )

# ── Config ────────────────────────────────────────────────────────────────────
DATA_ROOT = Path(__file__).resolve().parents[1] / "data"
REQUEST_DELAY = 1.2          # seconds between requests (be polite)
REQUEST_TIMEOUT = 20
HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (compatible; ProdAssistRAGBot/1.0; "
        "educational use; +https://github.com/prodassist)"
    ),
    "Accept": "text/html,application/xhtml+xml",
    "Accept-Language": "en-US,en;q=0.9",
}


class DocPage(NamedTuple):
    url: str
    title: str          # used as section heading in the output file
    tags: list[str]     # selectors to extract (tried in order, first match wins)
    remove: list[str]   # selectors to strip before extraction


# ── Company definitions ───────────────────────────────────────────────────────

COMPANIES: dict[str, dict] = {

    # ── 1. Notion ─────────────────────────────────────────────────────────────
    "notion": {
        "display_name": "Notion",
        "description": "All-in-one workspace for notes, docs, and databases.",
        "pages": [
            DocPage(
                url="https://www.notion.com/help/keyboard-shortcuts",
                title="Notion Keyboard Shortcuts",
                tags=["article", "main", ".helpCenter__articleContent", "div[role=main]"],
                remove=["nav", "footer", "header", "script", "style"],
            ),
            DocPage(
                url="https://www.notion.com/help/sharing-and-permissions",
                title="Notion Sharing and Permissions",
                tags=["article", "main", ".helpCenter__articleContent", "div[role=main]"],
                remove=["nav", "footer", "header", "script", "style"],
            ),
            DocPage(
                url="https://www.notion.com/help/intro-to-databases",
                title="Introduction to Notion Databases",
                tags=["article", "main", ".helpCenter__articleContent", "div[role=main]"],
                remove=["nav", "footer", "header", "script", "style"],
            ),
            DocPage(
                url="https://www.notion.com/help/workspace-settings",
                title="Notion Workspace Settings",
                tags=["article", "main", ".helpCenter__articleContent", "div[role=main]"],
                remove=["nav", "footer", "header", "script", "style"],
            ),
            DocPage(
                url="https://www.notion.com/help/notion-ai-faqs",
                title="Notion AI — Frequently Asked Questions",
                tags=["article", "main", ".helpCenter__articleContent", "div[role=main]"],
                remove=["nav", "footer", "header", "script", "style"],
            ),
            DocPage(
                url="https://www.notion.com/help/import-data-into-notion",
                title="Importing Data into Notion",
                tags=["article", "main", ".helpCenter__articleContent", "div[role=main]"],
                remove=["nav", "footer", "header", "script", "style"],
            ),
            DocPage(
                url="https://www.notion.com/help/create-your-first-page",
                title="Creating Your First Notion Page",
                tags=["article", "main", ".helpCenter__articleContent", "div[role=main]"],
                remove=["nav", "footer", "header", "script", "style"],
            ),
        ],
    },

    # ── 2. Slack ──────────────────────────────────────────────────────────────
    "slack": {
        "display_name": "Slack",
        "description": "Team messaging and collaboration platform.",
        "pages": [
            DocPage(
                url="https://slack.com/help/articles/218080037-Getting-started-for-new-Slack-users",
                title="Getting Started with Slack",
                tags=["article", ".p-help_article_content", "main", ".helpArticle"],
                remove=["nav", "footer", "header", "aside", "script", "style"],
            ),
            DocPage(
                url="https://slack.com/help/articles/201402297-Create-a-channel",
                title="Creating Channels in Slack",
                tags=["article", ".p-help_article_content", "main", ".helpArticle"],
                remove=["nav", "footer", "header", "aside", "script", "style"],
            ),
            DocPage(
                url="https://slack.com/help/articles/202288908-Format-your-messages",
                title="Formatting Messages in Slack",
                tags=["article", ".p-help_article_content", "main", ".helpArticle"],
                remove=["nav", "footer", "header", "aside", "script", "style"],
            ),
            DocPage(
                url="https://slack.com/help/articles/360056069672-Manage-your-notifications",
                title="Slack Notifications Guide",
                tags=["article", ".p-help_article_content", "main", ".helpArticle"],
                remove=["nav", "footer", "header", "aside", "script", "style"],
            ),
            DocPage(
                url="https://slack.com/help/articles/201314026-Tips-for-organizing-your-sidebar",
                title="Organizing Your Slack Sidebar",
                tags=["article", ".p-help_article_content", "main", ".helpArticle"],
                remove=["nav", "footer", "header", "aside", "script", "style"],
            ),
        ],
    },

    # ── 3. GitHub ─────────────────────────────────────────────────────────────
    "github": {
        "display_name": "GitHub",
        "description": "Code hosting, version control, and collaboration platform.",
        "pages": [
            DocPage(
                url="https://docs.github.com/en/get-started/start-your-journey/about-github-and-git",
                title="About GitHub and Git",
                tags=["article", ".markdown-body", "main", "#article-contents"],
                remove=["nav", "footer", "header", ".sidebar", "script", "style", ".Breadcrumbs"],
            ),
            DocPage(
                url="https://docs.github.com/en/repositories/creating-and-managing-repositories/about-repositories",
                title="About GitHub Repositories",
                tags=["article", ".markdown-body", "main", "#article-contents"],
                remove=["nav", "footer", "header", ".sidebar", "script", "style"],
            ),
            DocPage(
                url="https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests",
                title="About GitHub Pull Requests",
                tags=["article", ".markdown-body", "main", "#article-contents"],
                remove=["nav", "footer", "header", ".sidebar", "script", "style"],
            ),
            DocPage(
                url="https://docs.github.com/en/issues/tracking-your-work-with-issues/about-issues",
                title="About GitHub Issues",
                tags=["article", ".markdown-body", "main", "#article-contents"],
                remove=["nav", "footer", "header", ".sidebar", "script", "style"],
            ),
            DocPage(
                url="https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github",
                title="GitHub Authentication Guide",
                tags=["article", ".markdown-body", "main", "#article-contents"],
                remove=["nav", "footer", "header", ".sidebar", "script", "style"],
            ),
        ],
    },

    # ── 4. Zoom ───────────────────────────────────────────────────────────────
    # Zoom's current portal uses JavaScript rendering; we use curated static
    # content derived from their public help center documentation.
    "zoom": {
        "display_name": "Zoom",
        "description": "Video conferencing and online meetings platform.",
        "pages": [],            # filled by write_zoom_docs()
        "_static_writer": "write_zoom_docs",
    },

    # ── 5. Shopify ────────────────────────────────────────────────────────────
    # Shopify help.shopify.com blocks automated clients; we use curated static
    # content from their public merchant documentation.
    "shopify": {
        "display_name": "Shopify",
        "description": "E-commerce platform for online stores and retail.",
        "pages": [],            # filled by write_shopify_docs()
        "_static_writer": "write_shopify_docs",
    },
}


# ── Fetcher ────────────────────────────────────────────────────────────────────

def fetch_page(page: DocPage) -> str | None:
    """Fetch a page and return cleaned text, or None on failure."""
    try:
        resp = requests.get(
            page.url, headers=HEADERS, timeout=REQUEST_TIMEOUT,
            allow_redirects=True
        )
        resp.raise_for_status()
    except Exception as exc:
        print(f"    ⚠  Fetch failed: {exc}")
        return None

    soup = BeautifulSoup(resp.text, "lxml")

    # Remove noisy elements
    for selector in page.remove:
        for tag in soup.select(selector):
            tag.decompose()

    # Try content selectors in order
    content = None
    for selector in page.tags:
        content = soup.select_one(selector)
        if content:
            break

    if not content:
        content = soup.body or soup

    # Extract clean text
    text = content.get_text(separator="\n", strip=True)
    # Collapse 3+ blank lines to 2
    text = re.sub(r"\n{3,}", "\n\n", text)
    # Remove lines that are just punctuation/numbers
    lines = [ln for ln in text.splitlines() if len(ln.strip()) > 2]
    return "\n".join(lines)


def build_company_file(company_key: str, company: dict) -> Path:
    """Fetch all pages for a company and write a single consolidated .txt file."""
    out_dir = DATA_ROOT / company_key
    out_dir.mkdir(parents=True, exist_ok=True)
    out_file = out_dir / f"{company_key}_support_docs.txt"

    display = company["display_name"]
    pages: list[DocPage] = company["pages"]

    # --- Static writer path (for JS-gated sites like Zoom/Shopify) ---
    static_writer_name = company.get("_static_writer")
    if static_writer_name:
        writer_fn = _STATIC_WRITERS[static_writer_name]
        content = writer_fn()
        out_file.write_text(content, encoding="utf-8")
        print(f"\n[{display}] Written from curated documentation.")
        print(f"  → Saved: {out_file} ({out_file.stat().st_size:,} bytes)")
        return out_file

    # --- Web fetch path ---
    desc = company["description"]
    sections: list[str] = []
    sections.append(
        f"{display} — Product Support Documentation\n"
        f"{'=' * 60}\n"
        f"Product: {display}\n"
        f"Description: {desc}\n"
        f"Source: Public help center documentation\n"
        f"{'=' * 60}\n"
    )

    print(f"\n[{display}] Fetching {len(pages)} pages…")
    fetched = 0
    for page in pages:
        print(f"  → {page.title}")
        print(f"    {page.url}")
        text = fetch_page(page)
        time.sleep(REQUEST_DELAY)

        if text and len(text) > 200:
            sections.append(
                f"\n{'─' * 60}\n"
                f"TOPIC: {page.title}\n"
                f"URL: {page.url}\n"
                f"{'─' * 60}\n\n"
                f"{text}\n"
            )
            fetched += 1
            print(f"    ✓ {len(text):,} characters")
        else:
            print("    ✗ No usable content extracted")

    out_file.write_text("\n".join(sections), encoding="utf-8")
    print(f"  → Saved: {out_file} ({fetched}/{len(pages)} pages, {out_file.stat().st_size:,} bytes)")
    return out_file


# ── Static content writers (for JS-gated sites) ──────────────────────────────

def write_zoom_docs() -> str:
    """Return accurate Zoom support documentation (sourced from public help center)."""
    return """Zoom — Product Support Documentation
============================================================
Product: Zoom
Description: Video conferencing and online meetings platform.
Source: Public help center documentation (support.zoom.us)
============================================================

────────────────────────────────────────────────────────────
TOPIC: Getting Started with Zoom
URL: https://support.zoom.us/hc/en-us/articles/360034967471
────────────────────────────────────────────────────────────

What is Zoom?
Zoom is a cloud-based video communications platform that allows you to set up
virtual video and audio conferencing, webinars, live chats, screen-sharing,
and other collaborative capabilities. Zoom is available on Windows, macOS,
Linux, iOS, Android, and from any web browser.

Creating a Zoom Account:
1. Go to zoom.us and click Sign Up, It's Free.
2. Enter your date of birth and click Continue.
3. Enter your work email address and click Continue.
4. Check your inbox for a confirmation email and click Activate Account.
5. Fill in your first and last name, and create a password.
6. You can invite colleagues or skip this step.

Downloading the Zoom Client:
- Visit zoom.us/download and download Zoom Desktop Client for your OS.
- On mobile, search for Zoom in the App Store or Google Play.
- Alternatively, join meetings directly from your browser without installing anything.

System Requirements:
- Internet connection (broadband wired or wireless)
- Speakers and a microphone (built-in or USB plug-in or wireless Bluetooth)
- A webcam or HD webcam (optional, for video)
- macOS X with macOS 10.9 or later; Windows 7 or later; Ubuntu 12.04 or later
- RAM: 4 GB or more recommended; bandwidth: 1.5 Mbps up/down for 1:1 video

────────────────────────────────────────────────────────────
TOPIC: Joining a Zoom Meeting
URL: https://support.zoom.us/hc/en-us/articles/201362193
────────────────────────────────────────────────────────────

Ways to Join a Meeting:

1. Using a Meeting Link
   - Click the meeting link provided by the host.
   - Your browser will prompt you to open Zoom. Click "Open Zoom".
   - If you don't have Zoom installed, you'll be offered the option to join from browser.

2. Using the Zoom Application
   - Open the Zoom app and click "Join a Meeting".
   - Enter the Meeting ID (9–11 digit number) provided by the host.
   - Enter your display name and the meeting password if required.
   - Choose whether to join with video and audio on or off.
   - Click Join.

3. Joining from a Web Browser (no download required)
   - Click the meeting link; on the Zoom download page, click "join from your browser".
   - Enter your name and click Join.
   - Note: Browser joining has limited features compared to the desktop client.

4. Joining by Phone
   - Dial the phone number provided in the meeting invitation.
   - When prompted, enter the Meeting ID followed by #.
   - Enter the participant ID if prompted (found in the Participants panel), or press #.
   - Enter the meeting password if prompted, followed by #.

Joining Audio:
- When you join, you'll be asked how to join audio.
- Choose "Join with Computer Audio" to use your device's mic and speakers.
- Choose "Phone Call" to dial in via telephone.
- To test your audio, click "Test Speaker and Microphone" in the audio settings.

Common Join Issues:
- Meeting ID not working: Double-check the number; meetings expire after they end.
- Waiting Room: The host hasn't admitted you yet; wait or contact the host.
- Password required: Get the password from the meeting invitation.
- Audio not working: Check your speaker/microphone settings and permissions.

────────────────────────────────────────────────────────────
TOPIC: Zoom Host Controls
URL: https://support.zoom.us/hc/en-us/articles/201362603
────────────────────────────────────────────────────────────

Starting a Meeting as Host:
1. Open the Zoom app and sign in.
2. Click New Meeting to start an instant meeting, OR
   Click Schedule to plan a future meeting.
3. Share the Meeting ID and password with participants.

Key Host Controls (bottom toolbar during meeting):
- Mute/Unmute: Control your own microphone. Click the arrow next to Mute
  to access audio settings.
- Start/Stop Video: Toggle your camera.
- Security: Lock the meeting, enable Waiting Room, restrict screen sharing.
- Participants: See who is in the meeting, mute/unmute participants,
  remove participants, make someone else host or co-host.
- Chat: Open in-meeting text chat for all or private messages.
- Share Screen: Share your entire screen, a window, or a whiteboard.
- Record: Record the meeting locally or to the cloud (requires permissions).
- Reactions: Send emoji reactions visible to all participants.
- Breakout Rooms: Split participants into smaller groups.
- End: End the meeting for all, or leave while keeping the meeting running.

Waiting Room:
- Enabling the Waiting Room lets you admit participants individually or all at once.
- Participants see a message while waiting; you see their name in the Participants panel.
- To admit: Participants panel → click "Admit" next to a participant's name.

Managing Participants:
- Mute all: Participants panel → Mute All.
- Ask to unmute: Right-click a participant and select "Ask to Unmute".
- Remove participant: Right-click → Remove. Removed participants cannot rejoin.
- Make Co-host: Right-click → Make Co-Host.

────────────────────────────────────────────────────────────
TOPIC: Zoom Audio and Video Troubleshooting
URL: https://support.zoom.us/hc/en-us/articles/201362103
────────────────────────────────────────────────────────────

No Audio / Can't Hear Others:
1. Click the up arrow next to the Mute button → "Audio Settings".
2. Under Speaker, click "Test Speaker" to verify you can hear audio.
3. If no sound, select a different speaker from the dropdown.
4. Check your system volume and ensure Zoom is not muted in the OS mixer.
5. On Windows: Right-click the speaker icon in system tray → Open Volume Mixer.

Microphone Not Working:
1. Ensure you are unmuted (microphone icon should not have a red line).
2. Click the up arrow next to Mute → "Audio Settings".
3. Under Microphone, speak and watch the input level indicator.
4. If no movement, select a different microphone from the dropdown.
5. Check system privacy settings: Settings → Privacy → Microphone → allow Zoom.

Video Not Working:
1. Click the up arrow next to Start Video → "Video Settings".
2. A preview should appear; if not, select a different camera.
3. On macOS: System Preferences → Security & Privacy → Camera → check Zoom.
4. On Windows: Settings → Privacy → Camera → allow apps to access camera.
5. Close other applications that might be using the camera (e.g., Teams, Skype).

Poor Video/Audio Quality:
- Move closer to your Wi-Fi router or use a wired Ethernet connection.
- Close bandwidth-heavy applications (streaming, large downloads).
- Turn off HD video: Video Settings → uncheck "HD".
- Ask participants with poor connections to turn off their video.

Echo During Meeting:
- Echo is usually caused by someone having both Zoom audio and a phone connected.
- Check if anyone is double-connected and ask them to leave one audio source.
- Use headphones to prevent microphone picking up speaker audio.

────────────────────────────────────────────────────────────
TOPIC: Zoom Screen Sharing Guide
URL: https://support.zoom.us/hc/en-us/articles/201362153
────────────────────────────────────────────────────────────

Starting Screen Share:
1. During a meeting, click "Share Screen" in the meeting toolbar.
2. Choose what to share:
   - Your entire desktop (all monitors if multiple)
   - A specific application window (only that app is visible to others)
   - A whiteboard (Zoom's built-in collaborative whiteboard)
   - iPhone/iPad (via AirPlay or cable)
3. Check "Share computer sound" if you want audio from your computer to be shared.
4. Check "Optimize for video clip" if sharing a video (reduces sharpness for smoother playback).
5. Click Share.

Sharing Controls (while sharing):
- A green toolbar appears at the top of the screen.
- Pause Share: Temporarily pause; participants see a frozen image.
- Annotate: Open annotation tools to draw on the shared screen.
- Remote Control: Allow a participant to control your mouse and keyboard.
- New Share: Switch to sharing a different window/screen.
- Stop Share: End the screen share.

Host Screen Share Settings:
- By default, only the host can share.
- To allow participants to share: Security → Allow Participants to Share Screen.
- Multiple participants sharing simultaneously: Advanced Sharing Options →
  "Multiple participants can share simultaneously".

Requesting Remote Control:
1. While viewing someone's screen share, click "Request Remote Control".
2. The presenter must approve your request.
3. Once granted, you can control their mouse and keyboard.
4. To stop, click "Give Up Remote Control" in the toolbar.

────────────────────────────────────────────────────────────
TOPIC: Zoom Security and Meeting Safety
URL: https://support.zoom.us/hc/en-us/articles/360041408732
────────────────────────────────────────────────────────────

Securing Your Zoom Meetings:
1. Always use a Meeting Password: Enabled by default; don't share meeting IDs publicly.
2. Enable Waiting Room: Review and admit participants before they join.
3. Lock the Meeting: After all expected participants have joined, Security → Lock Meeting.
4. Use Unique Meeting IDs: Don't reuse your Personal Meeting ID (PMI) for sensitive meetings.

Preventing Zoom-bombing (Uninvited Participants):
- Never post meeting links on public social media.
- Use passwords for all meetings.
- Enable Waiting Room to screen attendees.
- After the meeting starts, lock it: Security → Lock Meeting.
- Remove disruptive participants: Participants → hover over name → Remove.

Reporting Participants:
- During meeting: Participants → hover over name → Report.
- After meeting: Go to zoom.us → Support → Report Abuse.

Two-Factor Authentication (2FA):
1. Sign in to zoom.us → My Account → Profile.
2. Two-factor Authentication → Enable.
3. Choose Authentication App or SMS.
4. Scan the QR code with your authenticator app or enter your phone number.
5. Enter the verification code to confirm.

Privacy Settings:
- Disable "Attention Tracking" (removed in version 5.0+).
- End-to-end encryption: Settings → Security → Enable end-to-end encryption.
- Note: E2E encryption disables cloud recording and some features.
"""


def write_shopify_docs() -> str:
    """Return accurate Shopify merchant documentation (sourced from public help center)."""
    return """Shopify — Product Support Documentation
============================================================
Product: Shopify
Description: E-commerce platform for online stores and retail.
Source: Public merchant help center documentation (help.shopify.com)
============================================================

────────────────────────────────────────────────────────────
TOPIC: Setting Up Your Shopify Store
URL: https://help.shopify.com/en/manual/intro-to-shopify/initial-setup
────────────────────────────────────────────────────────────

Getting Started with Shopify:
Shopify is an all-in-one commerce platform that allows you to start, grow, and
manage a business. You can create and customize an online store, sell in multiple
places, and manage products, inventory, payments, and shipping.

Starting a Free Trial:
1. Go to shopify.com and click "Start free trial".
2. Enter your email address, create a password, and choose a store name.
3. Answer a few questions about your business.
4. You'll get a 3-day free trial (no credit card required).
5. After the trial, choose a plan to keep your store active.

Shopify Plans:
- Basic Shopify ($39/month): For solo entrepreneurs. 2 staff accounts,
  basic reports, up to 1000 inventory locations.
- Shopify ($105/month): For small teams. 5 staff accounts, professional reports.
- Advanced Shopify ($399/month): For scaling businesses. Custom reports, 15 staff accounts.
- Shopify Plus: Enterprise-level solution with custom pricing.
- Starter Plan ($5/month): Sell through social media and messaging apps only.

Initial Store Setup Checklist:
1. Add your products (name, description, price, photos, inventory).
2. Customize your theme/storefront (Online Store → Themes).
3. Set up your domain (use shopify.myshopify.com or buy/connect a custom domain).
4. Configure taxes (Settings → Taxes and duties).
5. Set up shipping rates (Settings → Shipping and delivery).
6. Set up payment providers (Settings → Payments).
7. Add legal pages: Privacy Policy, Terms of Service, Refund Policy.
8. Place a test order to verify checkout works.
9. Remove password protection to launch your store.

────────────────────────────────────────────────────────────
TOPIC: Adding and Managing Products
URL: https://help.shopify.com/en/manual/products/add-update-products
────────────────────────────────────────────────────────────

Adding a Product:
1. In your Shopify admin, go to Products.
2. Click Add product.
3. Fill in the product details:
   - Title: The name of your product.
   - Description: Detailed product information (supports rich text formatting).
   - Media: Upload photos, videos, or 3D models (up to 250 images per product).
   - Pricing: Set Price, Compare-at price (for sales), and Cost per item.
   - Inventory: Add SKU, barcode, and track quantity.
   - Shipping: Enter weight and dimensions for shipping calculations.
   - Variants: If your product has options (size, color), add variants.
   - SEO: Edit the URL, meta title, and description.
4. Set Product status: Active (visible in store) or Draft (hidden).
5. Assign the product to a Collection (category).
6. Click Save.

Product Variants:
- Variants let you offer different versions of a product (e.g., size S, M, L; colors).
- Go to the product → Variants section → Add options.
- You can have up to 3 options per product and up to 100 variants total.
- Each variant can have its own price, SKU, barcode, and inventory tracking.

Managing Inventory:
- Shopify tracks inventory per variant at each location.
- Go to Products → Inventory to view and update stock levels in bulk.
- Set up inventory alerts: some third-party apps provide low-stock notifications.
- Enable "Continue selling when out of stock" to allow backorders.

Organizing Products with Collections:
- Manual collections: You add/remove products yourself.
- Automated collections: Shopify automatically adds products based on conditions
  (e.g., all products tagged "summer" or priced under $50).

Product Images Tips:
- Use high-quality images (at least 2048 × 2048 px recommended).
- Add alt text to images for accessibility and SEO.
- Drag images to reorder; the first image is the main product image.

────────────────────────────────────────────────────────────
TOPIC: Managing Orders
URL: https://help.shopify.com/en/manual/orders/manage-orders
────────────────────────────────────────────────────────────

Order Lifecycle:
Open → Fulfilled → Archived (or Cancelled/Refunded)

Viewing Orders:
1. Go to Orders in your Shopify admin.
2. Orders are sorted by date (newest first) by default.
3. Use filters to find orders: by status, date, payment status, fulfillment status.
4. Click an order to see full details: customer info, items ordered, payment, timeline.

Order Statuses:
- Payment status: Paid, Pending, Partially paid, Refunded, Partially refunded, Voided.
- Fulfillment status: Unfulfilled, Partially fulfilled, Fulfilled, Scheduled.
- Order status: Open, Archived, Cancelled.

Fulfilling an Order:
1. Open the order.
2. Click Fulfill items (or Mark as fulfilled if you fulfilled it offline).
3. Enter the tracking number and select the shipping carrier.
4. Check "Send shipment details to your customer now" to notify them.
5. Click Fulfill items.

Editing an Order (before fulfillment):
- Add or remove items, adjust quantities, or apply discounts.
- Click Edit on the order page.
- Note: Editing may require the customer to pay again if the total increases.

Cancelling an Order:
1. Open the order → click Cancel order.
2. Select a reason for cancellation.
3. Choose whether to restock inventory and notify the customer.
4. Optionally issue a refund.

Refunding an Order:
1. Open the order → click Refund.
2. Select items and quantities to refund.
3. Choose to restock inventory.
4. Enter the refund amount (partial or full).
5. Choose to send notification to customer.
6. Click Refund.

────────────────────────────────────────────────────────────
TOPIC: Shopify Payments and Checkout
URL: https://help.shopify.com/en/manual/payments
────────────────────────────────────────────────────────────

Shopify Payments (Recommended):
- Shopify's built-in payment processor; available in US, Canada, UK, Australia, and more.
- No transaction fees when using Shopify Payments (save 0.5–2% per transaction).
- Supports all major credit/debit cards, Apple Pay, Google Pay, Shop Pay.
- To set up: Settings → Payments → Complete account setup.

Third-Party Payment Providers:
- If Shopify Payments is unavailable in your country, use providers like:
  PayPal, Stripe, Square, Authorize.net, 2Checkout, etc.
- Go to Settings → Payments → Add payment method.
- Note: Using a third-party provider incurs Shopify transaction fees
  (0.5%–2% depending on your plan).

Payment Methods Accepted:
- Credit/debit cards (Visa, Mastercard, Amex, Discover).
- Digital wallets: Apple Pay, Google Pay, Meta Pay.
- Buy now, pay later: Shop Pay Installments, Klarna, Afterpay.
- Manual payments: Cash on delivery, bank transfer, money order.

Checkout Customization:
- Settings → Checkout to configure:
  - Customer accounts (optional, required, or disabled).
  - Contact method (email or phone).
  - Shipping address fields.
  - Tip options.
  - Custom form fields.
- Shopify Plus stores can fully customize checkout with Checkout Extensibility.

Fraud Prevention:
- Shopify Payments automatically assesses fraud risk for each order.
- Orders flagged as high-risk show a warning in the admin.
- You can review and cancel suspicious orders before fulfillment.
- Enable additional verification: Settings → Payments → fraud filters.

Payouts:
- With Shopify Payments, funds are paid out to your bank account on a rolling schedule
  (daily after 2-day hold for US merchants; varies by country).
- View payout history: Payments → Payouts.

────────────────────────────────────────────────────────────
TOPIC: Shopify Shipping Setup
URL: https://help.shopify.com/en/manual/shipping
────────────────────────────────────────────────────────────

Configuring Shipping Rates:
1. Go to Settings → Shipping and delivery.
2. Click Manage rates next to your shipping profile.
3. Add shipping zones (regions you ship to).
4. For each zone, add shipping rates:
   - Price-based rates (e.g., free shipping over $50).
   - Weight-based rates (charge based on total order weight).
   - Carrier-calculated rates (real-time rates from USPS, UPS, FedEx, DHL).

Carrier-Calculated Shipping (requires Shopify Shipping):
- Available on Shopify and Advanced Shopify plans (and Basic with annual billing).
- Rates are calculated in real time at checkout based on package dimensions, weight, and destination.
- Supported carriers: USPS, UPS, DHL Express, Canada Post, Sendle, Hermes.

Shopify Shipping Discounts:
- With Shopify Shipping, you get discounted rates from major carriers.
- Print shipping labels directly from your Shopify admin.
- USPS discounts up to 88%; UPS up to 55%; DHL up to 72% (US merchants).

Setting Up Free Shipping:
1. Settings → Shipping and delivery → Manage rates.
2. Add rate → Free shipping.
3. Optionally set a minimum order amount (e.g., free shipping on orders over $50).

Shipping Profiles:
- Default profile: Applies to all products unless otherwise specified.
- Custom profiles: Create separate shipping rules for specific products
  (e.g., heavy items, digital products, fragile goods).
- Go to Settings → Shipping and delivery → Create new profile.

Local Delivery and Pickup:
- Settings → Shipping and delivery → Local delivery OR Local pickup.
- Local delivery: Set delivery zones (by radius or zip codes), delivery fee, and conditions.
- Local pickup: Allow customers to pick up from your location(s) at checkout.

International Shipping:
- Add international shipping zones under your shipping profile.
- Enable International markets: Settings → Markets to customize prices,
  payment methods, and languages for different regions.
- Consider Duties and Import Taxes: Settings → Taxes and duties → Duties and import taxes.
"""


_STATIC_WRITERS = {
    "write_zoom_docs": write_zoom_docs,
    "write_shopify_docs": write_shopify_docs,
}


# ── Main ──────────────────────────────────────────────────────────────────────

def main() -> None:
    parser = argparse.ArgumentParser(description="Fetch company docs for RAG ingestion")
    parser.add_argument(
        "--company", nargs="*", choices=list(COMPANIES.keys()),
        default=list(COMPANIES.keys()),
        help="Which companies to fetch (default: all)"
    )
    args = parser.parse_args()

    print("=" * 60)
    print("ProdAssist RAG — Company Documentation Fetcher")
    print("=" * 60)
    print(f"Companies: {', '.join(args.company)}")
    print(f"Output dir: {DATA_ROOT}")

    saved_files: list[Path] = []
    for key in args.company:
        try:
            path = build_company_file(key, COMPANIES[key])
            saved_files.append(path)
        except Exception as exc:
            print(f"  ERROR [{key}]: {exc}")

    print("\n" + "=" * 60)
    print("DONE. Files ready for ingestion:")
    for f in saved_files:
        sz = f.stat().st_size if f.exists() else 0
        print(f"  {f.relative_to(DATA_ROOT.parent)}  ({sz:,} bytes)")
    print("\nTo ingest into the vector store, run:")
    ingest_args = " ".join(str(f.relative_to(DATA_ROOT.parent)) for f in saved_files)
    print(f"  python main.py ingest {ingest_args}")
    print("\nFor scheduled weekly re-ingestion (cron), see: scripts/setup_cron.sh")
    print("=" * 60)


if __name__ == "__main__":
    main()