|
|
|
|
|
"""
|
|
|
Setup script for the Document Text Extraction system.
|
|
|
Creates directories, checks dependencies, and initializes the project.
|
|
|
"""
|
|
|
|
|
|
import os
|
|
|
import sys
|
|
|
import subprocess
|
|
|
from pathlib import Path
|
|
|
import importlib.util
|
|
|
|
|
|
|
|
|
def check_python_version():
|
|
|
"""Check if Python version is compatible."""
|
|
|
if sys.version_info < (3, 8):
|
|
|
print("Python 3.8 or higher is required.")
|
|
|
print(f"Current version: {sys.version}")
|
|
|
return False
|
|
|
|
|
|
print(f"Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}")
|
|
|
return True
|
|
|
|
|
|
|
|
|
def create_directories():
|
|
|
"""Create necessary project directories."""
|
|
|
directories = [
|
|
|
"data/raw",
|
|
|
"data/processed",
|
|
|
"models",
|
|
|
"results/plots",
|
|
|
"results/metrics",
|
|
|
"logs"
|
|
|
]
|
|
|
|
|
|
print("\n📁 Creating project directories...")
|
|
|
for directory in directories:
|
|
|
Path(directory).mkdir(parents=True, exist_ok=True)
|
|
|
print(f" {directory}")
|
|
|
|
|
|
|
|
|
def check_dependencies():
|
|
|
"""Check if required dependencies are installed."""
|
|
|
print("\n📦 Checking dependencies...")
|
|
|
|
|
|
required_packages = [
|
|
|
('torch', 'PyTorch'),
|
|
|
('transformers', 'Transformers'),
|
|
|
('PIL', 'Pillow'),
|
|
|
('cv2', 'OpenCV'),
|
|
|
('pandas', 'Pandas'),
|
|
|
('numpy', 'NumPy'),
|
|
|
('sklearn', 'Scikit-learn')
|
|
|
]
|
|
|
|
|
|
missing_packages = []
|
|
|
|
|
|
for package, name in required_packages:
|
|
|
spec = importlib.util.find_spec(package)
|
|
|
if spec is None:
|
|
|
missing_packages.append(name)
|
|
|
print(f" {name} not found")
|
|
|
else:
|
|
|
print(f" {name}")
|
|
|
|
|
|
return missing_packages
|
|
|
|
|
|
|
|
|
def check_ocr_dependencies():
|
|
|
"""Check OCR-related dependencies."""
|
|
|
print("\nChecking OCR dependencies...")
|
|
|
|
|
|
|
|
|
try:
|
|
|
import easyocr
|
|
|
print(" EasyOCR")
|
|
|
except ImportError:
|
|
|
print(" EasyOCR not found")
|
|
|
|
|
|
|
|
|
try:
|
|
|
import pytesseract
|
|
|
print(" PyTesseract")
|
|
|
|
|
|
|
|
|
try:
|
|
|
pytesseract.get_tesseract_version()
|
|
|
print(" Tesseract OCR engine")
|
|
|
except Exception:
|
|
|
print(" Tesseract OCR engine not found or not in PATH")
|
|
|
print(" Please install Tesseract OCR:")
|
|
|
print(" - Windows: https://github.com/UB-Mannheim/tesseract/wiki")
|
|
|
print(" - Ubuntu: sudo apt install tesseract-ocr")
|
|
|
print(" - macOS: brew install tesseract")
|
|
|
|
|
|
except ImportError:
|
|
|
print(" PyTesseract not found")
|
|
|
|
|
|
|
|
|
def install_dependencies():
|
|
|
"""Install missing dependencies."""
|
|
|
print("\nInstalling dependencies from requirements.txt...")
|
|
|
|
|
|
try:
|
|
|
result = subprocess.run([
|
|
|
sys.executable, "-m", "pip", "install", "-r", "requirements.txt"
|
|
|
], capture_output=True, text=True, check=True)
|
|
|
|
|
|
print(" Dependencies installed successfully")
|
|
|
return True
|
|
|
|
|
|
except subprocess.CalledProcessError as e:
|
|
|
print(f" Failed to install dependencies: {e}")
|
|
|
print(f" Output: {e.stdout}")
|
|
|
print(f" Error: {e.stderr}")
|
|
|
return False
|
|
|
|
|
|
|
|
|
def check_gpu_support():
|
|
|
"""Check if GPU support is available."""
|
|
|
print("\n🖥️ Checking GPU support...")
|
|
|
|
|
|
try:
|
|
|
import torch
|
|
|
if torch.cuda.is_available():
|
|
|
gpu_count = torch.cuda.device_count()
|
|
|
gpu_name = torch.cuda.get_device_name(0)
|
|
|
print(f" CUDA available - {gpu_count} GPU(s)")
|
|
|
print(f" Primary GPU: {gpu_name}")
|
|
|
else:
|
|
|
print(" CUDA not available - will use CPU")
|
|
|
except ImportError:
|
|
|
print(" PyTorch not installed")
|
|
|
|
|
|
|
|
|
def create_sample_documents():
|
|
|
"""Create sample documents for testing."""
|
|
|
print("\nCreating sample test documents...")
|
|
|
|
|
|
sample_texts = [
|
|
|
"Invoice sent to John Doe on 01/15/2025\nInvoice No: INV-1001\nAmount: $1,500.00\nPhone: (555) 123-4567",
|
|
|
"Bill for Dr. Sarah Johnson dated March 10, 2025.\nInvoice Number: BL-2045.\nTotal: $2,300.50\nEmail: sarah@email.com",
|
|
|
"Receipt for Michael Brown\n456 Oak Street, Boston MA 02101\nInvoice: REC-3089\nDate: 2025-04-22\nAmount: $890.75"
|
|
|
]
|
|
|
|
|
|
sample_dir = Path("data/raw/samples")
|
|
|
sample_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
for i, text in enumerate(sample_texts, 1):
|
|
|
sample_file = sample_dir / f"sample_document_{i}.txt"
|
|
|
with open(sample_file, 'w', encoding='utf-8') as f:
|
|
|
f.write(text)
|
|
|
print(f" {sample_file.name}")
|
|
|
|
|
|
|
|
|
def run_initial_test():
|
|
|
"""Run a basic test to verify setup."""
|
|
|
print("\nRunning initial setup test...")
|
|
|
|
|
|
try:
|
|
|
|
|
|
from src.data_preparation import DocumentProcessor, NERDatasetCreator
|
|
|
from src.model import ModelConfig
|
|
|
print(" Core modules imported successfully")
|
|
|
|
|
|
|
|
|
processor = DocumentProcessor()
|
|
|
test_text = "Invoice sent to John Doe on 01/15/2025 Amount: $500.00"
|
|
|
cleaned_text = processor.clean_text(test_text)
|
|
|
print(" Document processor working")
|
|
|
|
|
|
|
|
|
dataset_creator = NERDatasetCreator(processor)
|
|
|
sample_dataset = dataset_creator.create_sample_dataset()
|
|
|
print(f" Dataset creator working - {len(sample_dataset)} samples")
|
|
|
|
|
|
|
|
|
config = ModelConfig()
|
|
|
print(f" Model config created - {config.num_labels} labels")
|
|
|
|
|
|
return True
|
|
|
|
|
|
except Exception as e:
|
|
|
print(f" Setup test failed: {e}")
|
|
|
return False
|
|
|
|
|
|
|
|
|
def display_next_steps():
|
|
|
"""Display next steps for the user."""
|
|
|
print("\n" + "=" * 30)
|
|
|
print("SETUP COMPLETED SUCCESSFULLY!")
|
|
|
print("=" * 30)
|
|
|
|
|
|
print("\nNext Steps:")
|
|
|
print("1. Quick Demo:")
|
|
|
print(" python demo.py")
|
|
|
|
|
|
print("\n2. Train Your Model:")
|
|
|
print(" # Add your documents to data/raw/")
|
|
|
print(" # Then run:")
|
|
|
print(" python src/training_pipeline.py")
|
|
|
|
|
|
print("\n3. 🌐 Start Web API:")
|
|
|
print(" python api/app.py")
|
|
|
print(" # Then open: http://localhost:8000")
|
|
|
|
|
|
print("\n4. Run Tests:")
|
|
|
print(" python tests/test_extraction.py")
|
|
|
|
|
|
print("\n5. 📚 Documentation:")
|
|
|
print(" # View README.md for detailed usage")
|
|
|
print(" # API docs: http://localhost:8000/docs")
|
|
|
|
|
|
print("\nPro Tips:")
|
|
|
print(" - Place your documents in data/raw/ for training")
|
|
|
print(" - Use GPU for faster training (if available)")
|
|
|
print(" - Adjust batch_size in config if you get memory errors")
|
|
|
print(" - Check logs/ directory for debugging information")
|
|
|
|
|
|
|
|
|
def main():
|
|
|
"""Main setup function."""
|
|
|
print("DOCUMENT TEXT EXTRACTION - SETUP SCRIPT")
|
|
|
print("=" * 60)
|
|
|
|
|
|
|
|
|
if not check_python_version():
|
|
|
return False
|
|
|
|
|
|
|
|
|
create_directories()
|
|
|
|
|
|
|
|
|
missing_packages = check_dependencies()
|
|
|
if missing_packages:
|
|
|
print(f"\nMissing packages: {', '.join(missing_packages)}")
|
|
|
install_deps = input("Install missing dependencies? (y/n): ").lower().strip()
|
|
|
|
|
|
if install_deps == 'y':
|
|
|
if not install_dependencies():
|
|
|
print("Failed to install dependencies. Please install manually:")
|
|
|
print(" pip install -r requirements.txt")
|
|
|
return False
|
|
|
else:
|
|
|
print("Some features may not work without required dependencies.")
|
|
|
|
|
|
|
|
|
check_ocr_dependencies()
|
|
|
|
|
|
|
|
|
check_gpu_support()
|
|
|
|
|
|
|
|
|
create_sample_documents()
|
|
|
|
|
|
|
|
|
if not run_initial_test():
|
|
|
print("Setup test failed. Some features may not work correctly.")
|
|
|
print(" Check error messages above and ensure all dependencies are installed.")
|
|
|
|
|
|
|
|
|
display_next_steps()
|
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
success = main()
|
|
|
|
|
|
if success:
|
|
|
print(f"\nSetup completed! Ready to extract text from documents!")
|
|
|
else:
|
|
|
print(f"\nSetup encountered issues. Please check the messages above.")
|
|
|
sys.exit(1) |
