main.py

# main.py

# --- CRITICAL FIX FOR CHROMADB SQLITE VERSION IN SUBPROCESS ---
import sys
try:
    __import__('pysqlite3')
    sys.modules['sqlite3'] = sys.modules.pop('pysqlite3')
except ImportError:
    pass
# --------------------------------------------------------------

import os
import argparse
from config import OUTPUT_DIR, CRAWL_CACHE_DIR, KNOWLEDGE_CACHE_DIR

# Import from the new modules
from web_research import run_web_analyst_mode, run_calendar_crawl_mode
from prescraped_processing import run_default_websites_mode

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Crawl4AI Unified Agent Backend")
    parser.add_argument("--mode", type=str, required=True, choices=['web_analyst', 'calendar_crawl', 'default_websites'], help="The agent mode to run.")
    parser.add_argument("--output-dir", type=str, default=OUTPUT_DIR, help="Directory to save output files.")
    parser.add_argument("--input-file", type=str, help="Path to a single input file (for web_analyst mode).")
    parser.add_argument("--input-files", nargs='+', help="List of input files (for default_websites mode).")
    parser.add_argument("--max-events", type=int, help="Maximum number of events to process.")
    parser.add_argument("--output-filename", type=str, help="Custom name for the output file (without extension).")
    
    # NEW ARGUMENT FOR FALLBACK
    parser.add_argument("--enable-fallback", action="store_true", help="Enable local schema fallback if DB2 fields are missing.")
    
    args = parser.parse_args()

    # Ensure directories exist
    for dir_path in[args.output_dir, CRAWL_CACHE_DIR, KNOWLEDGE_CACHE_DIR]:
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

    if args.mode == 'web_analyst':
        if not args.input_file:
            print("[ERROR] --input-file is required for web_analyst mode.")
        else:
            run_web_analyst_mode(
                input_file_path=args.input_file, 
                output_dir_override=args.output_dir, 
                max_events=args.max_events, 
                output_filename=args.output_filename,
                enable_fallback=args.enable_fallback
            )
    elif args.mode == 'calendar_crawl':
        run_calendar_crawl_mode(
            output_dir_override=args.output_dir, 
            max_events=args.max_events, 
            output_filename=args.output_filename,
            enable_fallback=args.enable_fallback
        )
    elif args.mode == 'default_websites':
        if not args.input_files:
            print("[ERROR] --input-files are required for default_websites mode.")
        else:
            run_default_websites_mode(
                input_files=args.input_files, 
                output_dir_override=args.output_dir, 
                max_events=args.max_events, 
                output_filename=args.output_filename,
                enable_fallback=args.enable_fallback
            )