Upload 9 files

.dockerignore

@@ -0,0 +1,52 @@
+# Git
+.git
+.gitignore
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual Environment
+venv/
+ENV/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# OS specific
+.DS_Store
+Thumbs.db
+
+# Docker
+.dockerignore
+docker-compose.yml
+
+# Large files and directories
+*.model
+*.bin
+*.h5
+*.ckpt
+*.pt
+*.pth
+models/

Dockerfile

@@ -0,0 +1,46 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+# Create a non-root user to run the application
+RUN groupadd -r appuser && useradd -r -g appuser appuser
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create cache directories with proper permissions
+RUN mkdir -p /app/model_cache && \
+    chmod 777 /app/model_cache
+
+# Copy requirements first for better caching
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . .
+COPY ../utils /app/utils
+COPY ../recommender.py /app/recommender.py
+
+# Set proper permissions
+RUN chown -R appuser:appuser /app
+
+# Expose port for FastAPI
+EXPOSE 7860
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+ENV TRANSFORMERS_CACHE=/app/model_cache
+ENV HF_HOME=/app/model_cache
+ENV SENTENCE_TRANSFORMERS_HOME=/app/model_cache
+
+# Switch to non-root user
+USER appuser
+
+# Command to run the application
+CMD ["python", "main.py"]

app.py

@@ -0,0 +1,138 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from typing import List
+import uvicorn
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from recommender import SHLRecommender
+from utils.validators import url as is_valid_url
+
+app = FastAPI(
+    title="SHL Test Recommender API",
+    description="API for recommending SHL tests based on job descriptions or queries",
+    version="1.0.0",
+    docs_url="/docs",
+    redoc_url="/redoc"
+)
+
+# Add CORS middleware to allow requests from any origin
+from fastapi.middleware.cors import CORSMiddleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Allow all origins
+    allow_credentials=True,
+    allow_methods=["*"],  # Allow all methods
+    allow_headers=["*"],  # Allow all headers
+)
+
+recommender = SHLRecommender()
+
+# Define request and response models
+class RecommendRequest(BaseModel):
+    query: str
+    max_recommendations: int = 10
+
+class Assessment(BaseModel):
+    url: str
+    adaptive_support: str
+    description: str
+    duration: int
+    remote_support: str
+    test_type: List[str]
+
+class RecommendationResponse(BaseModel):
+    recommended_assessments: List[Assessment]
+
+# API endpoints
+@app.get("/health")
+async def health_check():
+    try:
+        if not recommender or not hasattr(recommender, 'df') or recommender.df.empty:
+            return {"status": "unhealthy"}
+
+        if not hasattr(recommender, 'embedding_model') or not hasattr(recommender, 'model') or not hasattr(recommender, 'tokenizer'):
+            return {"status": "unhealthy"}
+
+        if not hasattr(recommender, 'product_embeddings') or len(recommender.product_embeddings) == 0:
+            return {"status": "unhealthy"}
+
+        return {"status": "healthy"}
+    except Exception:
+        return {"status": "unhealthy"}
+
+@app.get("/")
+async def root():
+    return {"message": "Welcome to the SHL Test Recommender API."}
+
+@app.post("/optimize")
+async def optimize_memory():
+    try:
+        recommender.optimize_memory()
+        return {"status": "success", "message": "Memory optimized successfully"}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Main recommend endpoint
+@app.post("/recommend", response_model=RecommendationResponse)
+async def recommend(request: RecommendRequest):
+    return await process_recommendation(request.query, request.max_recommendations)
+
+
+async def process_recommendation(query: str, max_recommendations: int):
+    try:
+        is_url = is_valid_url(query)
+
+        recommendations = recommender.get_recommendations(
+            query,
+            is_url=is_url,
+            max_recommendations=max_recommendations
+        )
+
+        formatted_assessments = []
+        for rec in recommendations:
+            duration_str = rec['Duration']
+            try:
+                duration_int = int(''.join(filter(str.isdigit, duration_str)))
+            except:
+                duration_int = 60
+
+            test_type_list = [rec['Test Type']] if rec['Test Type'] and rec['Test Type'] != "Unknown" else ["General Assessment"]
+
+            test_description = recommender.generate_test_description(
+                test_name=rec['Test Name'],
+                test_type=rec['Test Type'] if rec['Test Type'] and rec['Test Type'] != "Unknown" else "General Assessment"
+            )
+
+            description = test_description
+
+            formatted_assessments.append(
+                Assessment(
+                    url=rec['Link'],
+                    adaptive_support="Yes" if rec['Adaptive/IRT'] == "Yes" else "No",
+                    description=description,
+                    duration=duration_int,
+                    remote_support="Yes" if rec['Remote Testing'] == "Yes" else "No",
+                    test_type=test_type_list
+                )
+            )
+
+        return RecommendationResponse(
+            recommended_assessments=formatted_assessments
+        )
+    except Exception as e:
+        try:
+            recommender.optimize_memory()
+        except:
+            pass
+        raise HTTPException(status_code=500, detail=str(e))
+
+if __name__ == "__main__":
+    # Check if running on Hugging Face Spaces
+    IS_HF_SPACE = os.environ.get('SPACE_ID') is not None
+    port = 7860 if IS_HF_SPACE else 8000
+    
+    print(f"Starting FastAPI server on port {port}")
+    uvicorn.run("app:app", host="0.0.0.0", port=port, reload=True)

main.py

@@ -0,0 +1,12 @@
+import os
+import uvicorn
+
+from app import app
+
+if __name__ == "__main__":
+    # Check if running on Hugging Face Spaces
+    IS_HF_SPACE = os.environ.get('SPACE_ID') is not None
+    port = 7860 if IS_HF_SPACE else 8000
+    
+    print(f"Starting SHL Test Recommender API on port {port}")
+    uvicorn.run(app, host="0.0.0.0", port=port)

requirements.txt

@@ -0,0 +1,36 @@
+fastapi
+uvicorn
+gradio==5.20.0
+transformers
+pandas
+scikit-learn
+sentence-transformers
+torch
+beautifulsoup4
+requests
+python-multipart
+huggingface-hub
+safetensors
+tqdm
+typing-extensions
+pydantic
+psutil
+accelerate
+numpy
+scipy
+joblib
+aiofiles
+anyio
+httpx
+jinja2
+markdown-it-py
+pillow
+pyyaml
+regex
+rich
+setuptools
+six
+sympy
+tokenizers
+tomlkit
+typing-inspection

utils/data.csv

@@ -0,0 +1,115 @@
+Test Name,Remote Testing (Yes/No),Adaptive/IRT (Yes/No),Test Type,Link,Duration
+Teller 7.0,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/teller-7-0/,20 minutes
+Contact Center Sales & Service + 8.0,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/contact-center-sales-and-service-8-0/,20 minutes
+Director - Short Form,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/director-short-form/,20 minutes
+"Bookkeeping, Accounting, Auditing Clerk Short Form",Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/bookkeeping-accounting-auditing-clerk-short-form/,20 minutes
+.NET MVC (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/net-mvc-new/,45 minutes
+Written Spanish,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/written-spanish/,20 minutes
+Workplace Safety - Individual 7.1 (Americas),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/workplace-safety-individual-7-1-solution/,20 minutes
+Contact Center Sales & Service 8.0,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/contact-center-sales-and-service-8-0-4268/,20 minutes
+Adobe Photoshop CC,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/adobe-photoshop-cc/,20 minutes
+Apprentice 8.0 Job Focused Assessment,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/apprentice-8-0-job-focused-assessment/,20 minutes
+Apprentice + 8.0 Job Focused Assessment,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/apprentice-8-0-job-focused-assessment-4261/,20 minutes
+Apache HBase (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/apache-hbase-new/,20 minutes
+.NET Framework 4.5,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/net-framework-4-5/,45 minutes
+Automata - Fix (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/automata-fix-new/,20 minutes
+Workplace Safety Solution,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/workplace-safety-solution/,20 minutes
+Workplace Safety - Team 7.1 (International),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/workplace-safety-team-7-1-solution/,20 minutes
+Global Skills Development Report,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/global-skills-development-report/,45 minutes
+Contact Center Team Lead/Coach - Short Form,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/contact-center-team-leadcoach-short-form/,20 minutes
+Entry Level Customer Service 7.1 (International),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/entry-level-customer-service-%28retail-and-cc%29-7-1/,20 minutes
+Written English v1,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/written-english-v1/,20 minutes
+AngularJS (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/angularjs-new/,45 minutes
+Accounts Payable (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/accounts-payable-new/,20 minutes
+360 Digital Report,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/360-digital-report/,20 minutes
+Contact Centre Agent Solution - UK,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/contact-centre-agent-solution-uk/,20 minutes
+.NET WPF (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/net-wpf-new/,45 minutes
+AI Skills,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/ai-skills/,20 minutes
+Workplace Safety - Team 7.0 Solution,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/workplace-safety-team-7-0-solution/,20 minutes
+Claims/Operations Supervisor Solution,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/claimsoperations-supervisor-solution/,20 minutes
+Event Sales Manager Solution,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/event-sales-manager-solution/,20 minutes
+Account Manager Solution,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/account-manager-solution/,20 minutes
+Agile Software Development,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/agile-software-development/,7 minutes
+Entry Level Sales Sift Out 7.1,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/entry-level-sales-sift-out-7-1/,20 minutes
+Transcriptionist Solution,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/transcriptionist-solution/,20 minutes
+Android Development (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/android-development-new/,45 minutes
+Agency Manager Solution,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/agency-manager-solution/,20 minutes
+Teller with Sales - Short Form,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/teller-with-sales-short-form/,20 minutes
+Workplace Safety - Team 7.1 (Americas),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/workplace-safety-team-7-1-%28americas%29/,20 minutes
+Contact Center Manager - Short Form,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/contact-center-manager-short-form/,20 minutes
+ASP.NET 4.5,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/asp-net-4-5/,45 minutes
+Bank Operations Supervisor - Short Form,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/bank-operations-supervisor-short-form/,20 minutes
+360° Multi-Rater Feedback System (MFS),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/360-multi-rater-feedback-system-mfs/,20 minutes
+ADO.NET (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/ado-net-new/,45 minutes
+Apache Hadoop (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/apache-hadoop-new/,20 minutes
+Executive - Short Form,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/executive-short-form/,20 minutes
+Entry level Sales 7.1 (International),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/entry-level-sales-7-1/,20 minutes
+Zabbix (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/zabbix-new/,20 minutes
+Customer Service - Short Form,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/customer-service-short-form/,20 minutes
+Customer Service - Short Form - UK,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/customer-service-short-form-uk/,20 minutes
+Entry Level Customer Service 7.1 (South Africa),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/entry-level-customer-service-7-1-%28south-africa%29/,20 minutes
+Automata - SQL (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/automata-sql-new/,45 minutes
+Entry level Sales 7.1 (Americas),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/entry-level-sales-7-1-%28americas%29/,20 minutes
+.NET WCF (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/net-wcf-new/,45 minutes
+Financial Professional - Short Form,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/financial-professional-short-form/,20 minutes
+Entry Level Cashier 7.1 (Americas),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/entry-level-cashier-7-1-%28americas%29/,20 minutes
+Branch Manager - Short Form,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/branch-manager-short-form/,20 minutes
+Administrative Professional - Short Form,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/administrative-professional-short-form/,20 minutes
+Cashier Solution,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/cashier-solution/,20 minutes
+Accounts Receivable (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/accounts-receivable-new/,20 minutes
+Aerospace Engineering (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/aerospace-engineering-new/,20 minutes
+Aeronautical Engineering (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/aeronautical-engineering-new/,20 minutes
+Accounts Receivable Simulation (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/accounts-receivable-simulation-new/,20 minutes
+Apache Spark (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/apache-spark-new/,20 minutes
+.NET MVVM (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/net-mvvm-new/,45 minutes
+Accounts Payable Simulation (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/accounts-payable-simulation-new/,20 minutes
+Entry Level Cashier 7.1 (International),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/entry-level-cashier-7-1-%28international%29/,20 minutes
+.NET XAML (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/net-xaml-new/,45 minutes
+Apache Hive (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/apache-hive-new/,20 minutes
+Assessment and Development Center Exercises,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/assessment-and-development-center-exercises/,45 minutes
+Bank Administrative Assistant - Short Form,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/bank-administrative-assistant-short-form/,20 minutes
+Agile Testing (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/agile-testing-new/,20 minutes
+Automata (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/automata-new/,20 minutes
+Angular 6 (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/angular-6-new/,45 minutes
+Bilingual Spanish Reservation Agent Solution,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/bilingual-spanish-reservation-agent-solution/,20 minutes
+Apache Pig (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/apache-pig-new/,20 minutes
+ASP .NET with C# (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/asp-net-with-c-new/,45 minutes
+Workplace Safety - Individual 7.0 Solution,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/workplace-safety-individual-7-0-solution/,20 minutes
+Entry Level Customer Service 7.1 (Americas),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/entry-level-customer-service-7-1-%28americas%29/,20 minutes
+Apache Hadoop Extensions (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/apache-hadoop-extensions-new/,20 minutes
+Adobe Experience Manager (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/adobe-experience-manager-new/,20 minutes
+District/Regional Manager Solution,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/districtregional-manager-solution/,20 minutes
+Contact Center Customer Service + 8.0,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/contact-center-customer-service-8-0/,20 minutes
+Bank Collections Agent - Short Form,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/bank-collections-agent-short-form/,20 minutes
+Contact Center Customer Service 8.0,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/contact-center-customer-service-8-0-4269/,20 minutes
+Amazon Web Services (AWS) Development (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/amazon-web-services-aws-development-new/,45 minutes
+Customer Service with Sales - Short Form,Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/customer-service-with-sales-short-form/,20 minutes
+Apache Kafka (New),Yes,Yes,Cognitive Ability,https://www.shl.com/solutions/products/product-catalog/view/apache-kafka-new/,20 minutes
+Behavioral Assessments,Unknown,Unknown,Personality Assessment,https://www.shl.com/solutions/products/assessments/,25 minutes
+Virtual Assessment & Development Centers,Unknown,Yes,Unknown,https://www.shl.com/solutions/products/assessments/,45 minutes
+Personality Assessments,Unknown,Unknown,Personality Assessment,https://www.shl.com/solutions/products/assessments/,25 minutes
+Cognitive Assessments,Unknown,Unknown,Cognitive Ability,https://www.shl.com/solutions/products/assessments/,20 minutes
+Skills and Simulations,Unknown,Unknown,Unknown,https://www.shl.com/solutions/products/assessments/,Unknown
+Job Focused Assessments,Unknown,Unknown,Unknown,https://www.shl.com/solutions/products/assessments/,Unknown
+verify g plus,Yes,Yes,Unknown,https://www.shl.com/assets/documents/rebranded-assets/product-factsheet-verify-g-plus.pdf,Unknown
+verify deductive reasoning,Yes,Yes,Cognitive Ability,https://www.shl.com/assets/documents/rebranded-assets/product-factsheet-verify-deductive-reasoning.pdf,20 minutes
+verify interactive inductive,Yes,Yes,Unknown,https://www.shl.com/assets/documents/rebranded-assets/product-factsheet-verify-interactive-inductive.pdf,20 minutes
+verify interactive numerical reasoning,Yes,Yes,Cognitive Ability,https://www.shl.com/assets/documents/rebranded-assets/product-factsheet-verify-interactive-numerical-reasoning.pdf,20 minutes
+SHL's Cognitive Assessments,Unknown,Unknown,Cognitive Ability,https://www.shl.com/solutions/products/assessments/cognitive-assessments/,20 minutes
+SHL Verify G+ Test,Unknown,Yes,Unknown,https://www.shl.com/assets/documents/rebranded-assets/product-factsheet-verify-g-plus.pdf,Unknown
+SHL Verify Deductive Reasoning,Unknown,Yes,Unknown,https://www.shl.com/assets/documents/rebranded-assets/product-factsheet-verify-deductive-reasoning.pdf,20 minutes
+SHL Verify Interactive – Inductive Reasoning,Unknown,Yes,Unknown,https://www.shl.com/assets/documents/rebranded-assets/product-factsheet-verify-interactive-inductive.pdf,20 minutes
+SHL Verify Interactive – Numerical Reasoning,Unknown,Yes,Unknown,https://www.shl.com/assets/documents/rebranded-assets/product-factsheet-verify-interactive-numerical-reasoning.pdf,20 minutes
+Our Assessment Science,Unknown,Unknown,Unknown,https://www.shl.com/solutions/products/assessments/cognitive-assessments/,Unknown
+Graduate Hiring,Unknown,Unknown,Unknown,https://www.shl.com/solutions/products/assessments/cognitive-assessments/,Unknown
+Technology Hiring,Unknown,Unknown,Unknown,https://www.shl.com/solutions/products/assessments/cognitive-assessments/,Unknown
+Competency Fit,Unknown,Unknown,Unknown,https://www.shl.com/solutions/products/assessments/cognitive-assessments/,Unknown
+SHL Mobilize,Unknown,Unknown,Unknown,https://www.shl.com/solutions/products/assessments/cognitive-assessments/,Unknown
+SHL Occupational Personality Questionnaire (OPQ),Unknown,Unknown,Personality Assessment,https://www.shl.com/solutions/products/assessments/personality-assessment/,25 minutes
+SHL Motivational Questionnaire (MQ),Unknown,Unknown,Unknown,https://www.shl.com/solutions/products/assessments/personality-assessment/,Unknown
+Technical Skills,Unknown,Unknown,Technical Skills,https://www.shl.com/solutions/products/assessments/skills-and-simulations/,45 minutes
+Coding Simulations,Unknown,Unknown,Technical Skills,https://www.shl.com/solutions/products/assessments/skills-and-simulations/,45 minutes
+Language Skills,Unknown,Unknown,Unknown,https://www.shl.com/solutions/products/assessments/skills-and-simulations/,Unknown
+Contact Center Simulation,Unknown,Unknown,Unknown,https://www.shl.com/solutions/products/assessments/skills-and-simulations/,Unknown
+Business Skills,Unknown,Unknown,Unknown,https://www.shl.com/solutions/products/assessments/skills-and-simulations/,Unknown
+"And many, many more!",Unknown,Unknown,Unknown,https://www.shl.com/solutions/products/assessments/skills-and-simulations/,Unknown

utils/shl_product_scraper.py

@@ -0,0 +1,787 @@
+import requests
+from bs4 import BeautifulSoup
+import csv
+import re
+import time
+from urllib.parse import urljoin
+
+def scrape_shl_products():
+    # URL to scrape
+    base_url = "https://www.shl.com"
+    catalog_url = "https://www.shl.com/solutions/products/product-catalog/"
+
+    # Send HTTP request with improved headers
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'Connection': 'keep-alive',
+        'Upgrade-Insecure-Requests': '1',
+        'Cache-Control': 'max-age=0'
+    }
+
+    try:
+        # First try to get the main product catalog page
+        print(f"Fetching main catalog: {catalog_url}")
+        response = requests.get(catalog_url, headers=headers)
+        response.raise_for_status()
+
+        soup = BeautifulSoup(response.content, 'html.parser')
+
+        # Look for the actual product catalog items
+        view_links = []
+
+        for link in soup.find_all('a', href=True):
+            href = link['href']
+            if '/product-catalog/view/' in href:
+                view_links.append(href)
+
+        print(f"Found {len(view_links)} product catalog view links")
+
+        # Get more detail pages by looking at pagination
+        pagination_links = []
+        for link in soup.find_all('a', href=True):
+            href = link['href']
+            if '/product-catalog/' in href and ('?start=' in href or '&start=' in href):
+                pagination_links.append(href)
+
+        if pagination_links:
+            print(f"Found {len(pagination_links)} pagination links")
+
+            # Process each pagination page
+            for page_url in pagination_links:
+                # Make sure the URL is absolute
+                if not page_url.startswith('http'):
+                    page_url = urljoin(base_url, page_url)
+
+                print(f"Fetching pagination page: {page_url}")
+                try:
+                    page_response = requests.get(page_url, headers=headers)
+                    page_response.raise_for_status()
+                    page_soup = BeautifulSoup(page_response.content, 'html.parser')
+
+                    # Find all product view links on this pagination page
+                    for link in page_soup.find_all('a', href=True):
+                        href = link['href']
+                        if '/product-catalog/view/' in href:
+                            view_links.append(href)
+
+                except Exception as e:
+                    print(f"Error fetching pagination page: {e}")
+
+                # Be nice to the server
+                time.sleep(1)
+
+        # Remove duplicates and ensure all links are absolute
+        view_links = list(set(view_links))
+        view_links = [urljoin(base_url, link) if not link.startswith('http') else link for link in view_links]
+
+        print(f"Found {len(view_links)} unique product view links")
+
+        # Now we'll scrape each individual product page
+        products = []
+
+        for i, product_url in enumerate(view_links):
+            print(f"Scraping product {i+1}/{len(view_links)}: {product_url}")
+
+            try:
+                # Delay between requests to be polite to the server
+                if i > 0:
+                    time.sleep(1)
+
+                product_response = requests.get(product_url, headers=headers)
+                product_response.raise_for_status()
+                product_soup = BeautifulSoup(product_response.content, 'html.parser')
+
+                # Extract product name - usually in the title or main heading
+                product_name = ""
+                # Try to get it from the title
+                title_tag = product_soup.find('title')
+                if title_tag:
+                    title_text = title_tag.text.strip()
+                    # Clean up title - often in format "Product Name | SHL"
+                    if '|' in title_text:
+                        product_name = title_text.split('|')[0].strip()
+
+                # If no name from title, try the H1
+                if not product_name:
+                    h1_tag = product_soup.find('h1')
+                    if h1_tag:
+                        product_name = h1_tag.text.strip()
+
+                # If still no name, extract from URL
+                if not product_name:
+                    url_parts = product_url.rstrip('/').split('/')
+                    product_name = url_parts[-1].replace('-', ' ').title()
+
+                # Get page content as text for analysis
+                page_text = product_soup.get_text().lower()
+
+                # Try to determine if remote testing is supported
+                remote_testing = "Unknown"
+                remote_terms = ['remote', 'online', 'virtual', 'internet', 'web-based', 'digital', 'web browser',
+                               'online platform', 'from anywhere', 'off-site', 'distance']
+                remote_phrases = [
+                    'take the test remotely', 'administer remotely', 'online assessment', 'digital delivery',
+                    'web-based platform', 'remote proctoring', 'internet connection', 'browser-based',
+                    'accessible anywhere', 'remote testing', 'online testing'
+                ]
+
+                # Check for remote testing keywords
+                for term in remote_terms:
+                    if term in page_text:
+                        remote_testing = "Yes"
+                        break
+
+                # If not found with simple terms, check for phrases
+                if remote_testing == "Unknown":
+                    for phrase in remote_phrases:
+                        if phrase in page_text:
+                            remote_testing = "Yes"
+                            break
+
+                # Most modern SHL tests are remote, so if we're still uncertain, check for contrary evidence
+                if remote_testing == "Unknown" and not any(x in page_text for x in ['in-person only', 'on-site only', 'physical test center required']):
+                    # If product URL or name contains certain keywords, it's likely remote
+                    product_url_lower = product_url.lower()
+                    if any(x in product_url_lower for x in ['online', 'digital', 'remote', 'virtual']):
+                        remote_testing = "Yes"
+                    # For SHL products, most are remote unless explicitly stated otherwise
+                    elif 'shl.com' in product_url:
+                        remote_testing = "Yes"
+
+                # ENHANCED ADAPTIVE/IRT DETECTION
+                adaptive = "Unknown"
+                # Direct adaptive terminology
+                adaptive_terms = [
+                    'adaptive', 'irt', 'item response theory', 'tailored', 'adjusts difficulty',
+                    'computer adaptive', 'cat', 'adaptive testing', 'dynamic difficulty',
+                    'adjusts questions', 'personalized assessment', 'adaptive algorithm',
+                    'smart testing', 'tailored questioning', 'adaptive assessment'
+                ]
+
+                # Phrases that indicate adaptive testing
+                adaptive_phrases = [
+                    'questions adapt based on', 'difficulty adjusts', 'tailored to ability',
+                    'adapts to the test taker', 'customizes questions', 'dynamic question selection',
+                    'questions change based on previous answers', 'adapts to user performance',
+                    'intelligent testing algorithm', 'questions get harder or easier'
+                ]
+
+                # Check for adaptive terms
+                for term in adaptive_terms:
+                    if term in page_text:
+                        adaptive = "Yes"
+                        break
+
+                # If not found with simple terms, check for phrases
+                if adaptive == "Unknown":
+                    for phrase in adaptive_phrases:
+                        if phrase in page_text:
+                            adaptive = "Yes"
+                            break
+
+                # Check headings and specific sections that might contain information
+                if adaptive == "Unknown":
+                    for heading in product_soup.find_all(['h2', 'h3', 'h4']):
+                        heading_text = heading.get_text().lower()
+                        if any(term in heading_text for term in ['adaptive', 'test methodology', 'assessment technology']):
+                            # Get the next paragraph or content
+                            next_elem = heading.find_next(['p', 'div'])
+                            if next_elem:
+                                next_text = next_elem.get_text().lower()
+                                if any(term in next_text for term in adaptive_terms) or any(phrase in next_text for phrase in adaptive_phrases):
+                                    adaptive = "Yes"
+                                    break
+
+                # Look for specific product indicators
+                if adaptive == "Unknown":
+                    # Many SHL Verify tests are adaptive
+                    test_name_lower = product_name.lower()
+                    if 'verify' in test_name_lower and any(x in test_name_lower for x in ['reasoning', 'ability', 'numerical', 'verbal', 'logical']):
+                        adaptive = "Yes"
+                    # Also check for ADEPT or CAT in name
+                    elif any(x in test_name_lower for x in ['adept', 'cat']):
+                        adaptive = "Yes"
+
+                # Try to extract test type
+                test_type = "Unknown"
+                type_mapping = {
+                    "Cognitive Ability": ['cognitive', 'ability', 'intelligence', 'reasoning', 'numerical', 'verbal', 'logical'],
+                    "Personality Assessment": ['personality', 'behavioral', 'behaviour', 'character', 'temperament'],
+                    "Technical Skills": ['technical', 'coding', 'programming', 'development', 'software', 'microsoft', 'excel'],
+                    "Situational Judgment": ['situational', 'judgment', 'judgement', 'scenario', 'case study'],
+                    "Job-Specific Assessment": ['job-specific', 'role-specific', 'position', 'occupation']
+                }
+
+                for test_category, keywords in type_mapping.items():
+                    for keyword in keywords:
+                        if keyword in page_text:
+                            test_type = test_category
+                            break
+                    if test_type != "Unknown":
+                        break
+
+                # ENHANCED DURATION EXTRACTION
+                duration = "Unknown"
+
+                # Common duration pattern phrases to look for - expanded with more patterns
+                duration_patterns = [
+                    r'takes?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                    r'duration\s*:?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                    r'time\s*(?:limit|frame|allotted)?\s*:?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                    r'(?:test|assessment)\s*(?:duration|length|time)\s*:?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                    r'(?:takes?|requires?|needs?)\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                    r'completed in\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                    r'(?:typically|usually|generally|normally)\s*(?:takes?|lasts?|runs?)\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                    r'time\s*to\s*(?:complete|finish)\s*(?:is|:)?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                    r'(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                    r'(\d+)\s*(?:min|mins|minutes|minute)',
+                    r'time(?:frame)?:?\s*(\d+)',
+                    r'duration:?\s*(\d+)',
+                ]
+
+                # Specialized function to process duration matches
+                def process_duration_match(match):
+                    if match.group(2) and match.group(2).isdigit():
+                        # If there's a range, use the maximum value
+                        return f"{match.group(2)} minutes"
+                    else:
+                        return f"{match.group(1)} minutes"
+
+                # First look for duration in specific elements that are likely to contain duration info
+                duration_containers = product_soup.find_all(['li', 'p', 'span', 'div'], string=re.compile(
+                    r'(?:duration|time|minutes|mins|length|complete)', re.I))
+
+                for container in duration_containers:
+                    container_text = container.get_text().lower()
+                    for pattern in duration_patterns:
+                        duration_match = re.search(pattern, container_text)
+                        if duration_match:
+                            duration = process_duration_match(duration_match)
+                            break
+                    if duration != "Unknown":
+                        break
+
+                # If still unknown, look in tables that might contain specs or details
+                if duration == "Unknown":
+                    tables = product_soup.find_all('table')
+                    for table in tables:
+                        rows = table.find_all('tr')
+                        for row in rows:
+                            row_text = row.get_text().lower()
+                            if any(term in row_text for term in ['duration', 'time', 'minutes', 'length']):
+                                for pattern in duration_patterns:
+                                    duration_match = re.search(pattern, row_text)
+                                    if duration_match:
+                                        duration = process_duration_match(duration_match)
+                                        break
+                            if duration != "Unknown":
+                                break
+                        if duration != "Unknown":
+                            break
+
+                # If still unknown, search all text on the page
+                if duration == "Unknown":
+                    for pattern in duration_patterns:
+                        duration_match = re.search(pattern, page_text)
+                        if duration_match:
+                            duration = process_duration_match(duration_match)
+                            break
+
+                # Check for PDF links that might have test details
+                if duration == "Unknown" or adaptive == "Unknown":
+                    pdf_links = [a['href'] for a in product_soup.find_all('a', href=True) if a['href'].endswith('.pdf')]
+                    for pdf_link in pdf_links:
+                        # We'll just note that there's a PDF that might have more info
+                        pdf_url = urljoin(product_url, pdf_link)
+                        print(f"Found potential info PDF: {pdf_url}")
+                        # We don't download and parse PDFs here but could expand functionality
+
+                # If still unknown, assign duration based on product name and test type
+                if duration == "Unknown":
+                    test_name_lower = product_name.lower()
+
+                    # Technical and programming tests typically take longer
+                    if any(tech in test_name_lower for tech in [
+                        '.net', 'java', 'python', 'c#', 'javascript', 'angular', 'react',
+                        'node', 'aws', 'cloud', 'azure', 'devops', 'programming', 'coding',
+                        'development', 'sql', 'database'
+                    ]):
+                        duration = "45 minutes"
+
+                    # Cognitive tests have standard durations
+                    elif test_type == "Cognitive Ability":
+                        if any(term in test_name_lower for term in ['numerical', 'verbal']):
+                            duration = "20 minutes"
+                        elif 'inductive' in test_name_lower:
+                            duration = "25 minutes"
+                        elif 'deductive' in test_name_lower:
+                            duration = "20 minutes"
+                        elif 'reasoning' in test_name_lower:
+                            duration = "20 minutes"
+                        else:
+                            duration = "20 minutes"  # Default for cognitive tests
+
+                    # Personality assessments typically take 25-30 minutes
+                    elif test_type == "Personality Assessment":
+                        duration = "25 minutes"
+
+                    # SJTs typically take 30 minutes
+                    elif test_type == "Situational Judgment":
+                        duration = "30 minutes"
+
+                    # Short form assessments are usually shorter
+                    elif 'short form' in test_name_lower:
+                        duration = "15 minutes"
+
+                    # Check for specific product types in the name
+                    elif 'agile' in test_name_lower and 'software' in test_name_lower:
+                        duration = "7 minutes"  # From your data
+
+                products.append({
+                    "Test Name": product_name,
+                    "Remote Testing": remote_testing,
+                    "Adaptive/IRT": adaptive,
+                    "Test Type": test_type,
+                    "Link": product_url,
+                    "Duration": duration
+                })
+
+            except Exception as e:
+                print(f"Error scraping product {product_url}: {e}")
+
+        # If we have few or no products, we'll add the known SHL products
+        if len(products) < 10:
+            print("Adding known SHL products as fallback")
+            known_products = [
+                {
+                    "Test Name": "Verify G+ General Ability Test",
+                    "Remote Testing": "Yes",
+                    "Adaptive/IRT": "Yes",
+                    "Test Type": "Cognitive Ability",
+                    "Link": "https://www.shl.com/solutions/products/verify-g-general-ability-test/",
+                    "Duration": "18 minutes"
+                },
+                {
+                    "Test Name": "SHL Personality Inventory",
+                    "Remote Testing": "Yes",
+                    "Adaptive/IRT": "No",
+                    "Test Type": "Personality Assessment",
+                    "Link": "https://www.shl.com/solutions/products/personality-inventory/",
+                    "Duration": "25 minutes"
+                },
+                {
+                    "Test Name": "Verify Numerical Reasoning Test",
+                    "Remote Testing": "Yes",
+                    "Adaptive/IRT": "Yes",
+                    "Test Type": "Cognitive Ability",
+                    "Link": "https://www.shl.com/solutions/products/verify-numerical-reasoning-test/",
+                    "Duration": "15 minutes"
+                },
+                {
+                    "Test Name": "Verify Verbal Reasoning Test",
+                    "Remote Testing": "Yes",
+                    "Adaptive/IRT": "Yes",
+                    "Test Type": "Cognitive Ability",
+                    "Link": "https://www.shl.com/solutions/products/verify-verbal-reasoning-test/",
+                    "Duration": "15 minutes"
+                },
+                {
+                    "Test Name": "OPQ32 Occupational Personality Questionnaire",
+                    "Remote Testing": "Yes",
+                    "Adaptive/IRT": "No",
+                    "Test Type": "Personality Assessment",
+                    "Link": "https://www.shl.com/solutions/products/opq32-occupational-personality-questionnaire/",
+                    "Duration": "35 minutes"
+                },
+                {
+                    "Test Name": "Situational Judgment Test",
+                    "Remote Testing": "Yes",
+                    "Adaptive/IRT": "No",
+                    "Test Type": "Situational Judgment",
+                    "Link": "https://www.shl.com/solutions/products/situational-judgment-test/",
+                    "Duration": "30 minutes"
+                },
+                {
+                    "Test Name": "Coding Assessment",
+                    "Remote Testing": "Yes",
+                    "Adaptive/IRT": "No",
+                    "Test Type": "Technical Skills",
+                    "Link": "https://www.shl.com/solutions/products/coding-assessment/",
+                    "Duration": "60 minutes"
+                },
+                {
+                    "Test Name": "MQ Motivation Questionnaire",
+                    "Remote Testing": "Yes",
+                    "Adaptive/IRT": "No",
+                    "Test Type": "Motivation Assessment",
+                    "Link": "https://www.shl.com/solutions/products/mq-motivation-questionnaire/",
+                    "Duration": "25 minutes"
+                },
+                {
+                    "Test Name": "ADEPT-15 Personality Assessment",
+                    "Remote Testing": "Yes",
+                    "Adaptive/IRT": "Yes",
+                    "Test Type": "Personality Assessment",
+                    "Link": "https://www.shl.com/solutions/products/adept-15/",
+                    "Duration": "20 minutes"
+                },
+                {
+                    "Test Name": "Inductive Reasoning Test",
+                    "Remote Testing": "Yes",
+                    "Adaptive/IRT": "Yes",
+                    "Test Type": "Cognitive Ability",
+                    "Link": "https://www.shl.com/solutions/products/inductive-reasoning-test/",
+                    "Duration": "20 minutes"
+                },
+                {
+                    "Test Name": "Microsoft Office Assessment",
+                    "Remote Testing": "Yes",
+                    "Adaptive/IRT": "No",
+                    "Test Type": "Technical Skills",
+                    "Link": "https://www.shl.com/solutions/products/microsoft-office-assessment/",
+                    "Duration": "40 minutes"
+                },
+                {
+                    "Test Name": "Call Center Assessment",
+                    "Remote Testing": "Yes",
+                    "Adaptive/IRT": "No",
+                    "Test Type": "Job-Specific Assessment",
+                    "Link": "https://www.shl.com/solutions/products/call-center-assessment/",
+                    "Duration": "30 minutes"
+                }
+            ]
+
+            # Add known products that aren't already in our list
+            seen_names = set(product["Test Name"] for product in products)
+            for product in known_products:
+                if product["Test Name"] not in seen_names:
+                    products.append(product)
+
+        # Write data to CSV
+        with open('utils\data.csv', 'w', newline='', encoding='utf-8') as csvfile:
+            fieldnames = ["Test Name", "Remote Testing (Yes/No)", "Adaptive/IRT (Yes/No)", "Test Type", "Link", "Duration"]
+            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+
+            writer.writeheader()
+            for product in products:
+                writer.writerow({
+                    "Test Name": product["Test Name"],
+                    "Remote Testing (Yes/No)": product["Remote Testing"],
+                    "Adaptive/IRT (Yes/No)": product["Adaptive/IRT"],
+                    "Test Type": product["Test Type"],
+                    "Link": product["Link"],
+                    "Duration": product["Duration"]
+                })
+
+        print(f"Successfully scraped {len(products)} products and saved to data.csv")
+
+        # Also try to scrape additional product information from other SHL pages
+        try:
+            scrape_additional_products(headers, products, base_url)
+        except Exception as e:
+            print(f"Error during additional product scraping: {e}")
+
+        # Add an extra pass to improve duration and adaptive information
+        try:
+            enhance_product_information(products)
+        except Exception as e:
+            print(f"Error during information enhancement: {e}")
+
+    except requests.exceptions.RequestException as e:
+        print(f"Error fetching the URL: {e}")
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        import traceback
+        traceback.print_exc()
+
+def enhance_product_information(products):
+    """Add an additional pass to improve duration and adaptive/IRT information"""
+    print("Enhancing product information...")
+
+    # Define common test durations by product category or type
+    test_duration_mapping = {
+        # Technical/coding tests
+        'technical': {
+            'default': '45 minutes',
+            'keywords': ['.net', 'java', 'python', 'c#', 'javascript', 'angular', 'react',
+                        'node', 'aws', 'cloud', 'azure', 'devops', 'programming', 'coding',
+                        'development', 'sql', 'database', 'technical']
+        },
+        # Cognitive tests
+        'cognitive': {
+            'default': '20 minutes',
+            'keywords': ['cognitive', 'ability', 'reasoning', 'numerical', 'verbal', 'logical', 'inductive']
+        },
+        # Personality assessments
+        'personality': {
+            'default': '25 minutes',
+            'keywords': ['personality', 'behavioral', 'behaviour', 'character', 'temperament']
+        },
+        # Situational judgment tests
+        'situational': {
+            'default': '30 minutes',
+            'keywords': ['situational', 'judgment', 'judgement', 'scenario', 'case study']
+        }
+    }
+
+    # Enhanced list of products known to be adaptive
+    adaptive_products = [
+        'verify', 'adept', 'cat', 'adaptive', 'irt', 'g+', 'verify g', 'verify numerical',
+        'verify verbal', 'verify inductive', 'verify interactive'
+    ]
+
+    # Enhance each product
+    for product in products:
+        # First enhance duration information if it's Unknown
+        if product["Duration"] == "Unknown":
+            product_name_lower = product["Test Name"].lower()
+            test_type_lower = product["Test Type"].lower() if product["Test Type"] else ""
+
+            # Check for short form assessments
+            if 'short form' in product_name_lower:
+                product["Duration"] = "15 minutes"
+                continue
+
+            # Apply the mappings based on test name and type
+            for category, details in test_duration_mapping.items():
+                keywords = details['keywords']
+                if any(keyword in product_name_lower for keyword in keywords) or any(keyword in test_type_lower for keyword in keywords):
+                    product["Duration"] = details['default']
+                    break
+
+            # Special cases for specific products
+            if 'agile software development' in product_name_lower:
+                product["Duration"] = "7 minutes"
+
+        # Then enhance adaptive/IRT information if it's Unknown
+        if product["Adaptive/IRT"] == "Unknown":
+            product_name_lower = product["Test Name"].lower()
+
+            # Check if the product name contains any keywords associated with adaptive tests
+            if any(adaptive_term in product_name_lower for adaptive_term in adaptive_products):
+                product["Adaptive/IRT"] = "Yes"
+
+            # Specific product families known to use adaptive technology
+            elif product_name_lower.startswith('verify') and 'reasoning' in product_name_lower:
+                product["Adaptive/IRT"] = "Yes"
+            elif 'interactive' in product_name_lower and any(term in product_name_lower for term in ['reasoning', 'ability', 'cognitive']):
+                product["Adaptive/IRT"] = "Yes"
+
+    # Write the enhanced data back to CSV
+    with open('utils\data.csv', 'w', newline='', encoding='utf-8') as csvfile:
+        fieldnames = ["Test Name", "Remote Testing (Yes/No)", "Adaptive/IRT (Yes/No)", "Test Type", "Link", "Duration"]
+        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+
+        writer.writeheader()
+        for product in products:
+            writer.writerow({
+                "Test Name": product["Test Name"],
+                "Remote Testing (Yes/No)": product["Remote Testing"],
+                "Adaptive/IRT (Yes/No)": product["Adaptive/IRT"],
+                "Test Type": product["Test Type"],
+                "Link": product["Link"],
+                "Duration": product["Duration"]
+            })
+
+    print("Product information enhancement completed")
+
+def scrape_additional_products(headers, existing_products, base_url):
+    """Scrape additional product information from other SHL pages"""
+
+    # Additional pages that might have product information
+    additional_urls = [
+        "https://www.shl.com/solutions/products/assessments/",
+        "https://www.shl.com/solutions/products/assessments/cognitive-assessments/",
+        "https://www.shl.com/solutions/products/assessments/personality-assessment/",
+        "https://www.shl.com/solutions/products/assessments/skills-and-simulations/"
+    ]
+
+    seen_names = set(product["Test Name"] for product in existing_products)
+    new_products = []
+
+    for url in additional_urls:
+        print(f"Scraping additional products from: {url}")
+
+        try:
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()
+            soup = BeautifulSoup(response.content, 'html.parser')
+
+            # Look for PDF links that might contain detailed product information
+            pdf_links = [a['href'] for a in soup.find_all('a', href=True) if a['href'].endswith('.pdf')]
+
+            for pdf_link in pdf_links:
+                pdf_url = urljoin(base_url, pdf_link)
+                print(f"Found potential product PDF: {pdf_url}")
+
+                # Extract a possible product name from the PDF link
+                pdf_name = pdf_link.split('/')[-1].replace('-', ' ').replace('_', ' ').replace('.pdf', '')
+
+                # Clean up the name
+                pdf_name = re.sub(r'product factsheet', '', pdf_name, flags=re.IGNORECASE).strip()
+
+                # If it looks like a valid product name, add it
+                if len(pdf_name) > 3 and pdf_name not in seen_names:
+                    # Extract product details from the PDF name
+                    is_verify = 'verify' in pdf_name.lower()
+                    is_adaptive = is_verify or any(term in pdf_name.lower() for term in ['adaptive', 'interactive'])
+
+                    product_type = "Unknown"
+                    if any(term in pdf_name.lower() for term in ['reasoning', 'numerical', 'verbal', 'cognitive']):
+                        product_type = "Cognitive Ability"
+                    elif any(term in pdf_name.lower() for term in ['personality', 'behavioral']):
+                        product_type = "Personality Assessment"
+
+                    # Assign duration based on product type
+                    duration = "Unknown"
+                    if product_type == "Cognitive Ability":
+                        if 'numerical' in pdf_name.lower() or 'verbal' in pdf_name.lower():
+                            duration = "20 minutes"
+                        elif 'deductive' in pdf_name.lower():
+                            duration = "20 minutes"
+                        else:
+                            duration = "20 minutes"
+                    elif product_type == "Personality Assessment":
+                        duration = "25 minutes"
+
+                    new_products.append({
+                        "Test Name": pdf_name,
+                        "Remote Testing": "Yes",  # Modern SHL tests are typically remote
+                        "Adaptive/IRT": "Yes" if is_adaptive else "Unknown",
+                        "Test Type": product_type,
+                        "Link": pdf_url,
+                        "Duration": duration
+                    })
+
+                    seen_names.add(pdf_name)
+
+            # Find product sections - look for content blocks with headings followed by descriptions
+            sections = soup.find_all(['section', 'div'], class_=['product-section', 'content-block', 'product-listing'])
+
+            if not sections:
+                # If no obvious sections, look for headings that might describe products
+                headings = soup.find_all(['h2', 'h3'], class_=lambda c: c and ('title' in c or 'heading' in c))
+
+                for heading in headings:
+                    product_name = heading.get_text().strip()
+
+                    if len(product_name) < 5 or product_name in seen_names:
+                        continue
+
+                    # Find a nearby link
+                    parent = heading.find_parent()
+                    link_elem = parent.find('a') if parent else None
+                    product_url = link_elem['href'] if link_elem and link_elem.has_attr('href') else url
+
+                    if not product_url.startswith('http'):
+                        product_url = urljoin(base_url, product_url)
+
+                    # Get description
+                    description = ""
+                    next_elem = heading.find_next_sibling()
+                    if next_elem and next_elem.name == 'p':
+                        description = next_elem.get_text().lower()
+
+                    # Extract info from description
+                    remote_testing = "Yes" if any(term in description for term in ['remote', 'online', 'virtual']) else "Unknown"
+                    adaptive = "Yes" if any(term in description for term in ['adaptive', 'irt', 'tailored']) else "Unknown"
+
+                    # Determine test type
+                    test_type = "Unknown"
+                    if any(term in product_name.lower() or term in description for term in ['cognitive', 'ability', 'intelligence']):
+                        test_type = "Cognitive Ability"
+                    elif any(term in product_name.lower() or term in description for term in ['personality', 'behavioral']):
+                        test_type = "Personality Assessment"
+                    elif any(term in product_name.lower() or term in description for term in ['situational', 'judgment']):
+                        test_type = "Situational Judgment"
+                    elif any(term in product_name.lower() or term in description for term in ['coding', 'programming', 'technical']):
+                        test_type = "Technical Skills"
+
+                    # Look for duration with enhanced patterns
+                    duration = "Unknown"
+
+                    # Common duration pattern phrases to look for
+                    duration_patterns = [
+                        r'takes?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                        r'duration\s*:?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                        r'time\s*(?:limit|frame|allotted)?\s*:?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                        r'(?:test|assessment)\s*(?:duration|length|time)\s*:?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                        r'(?:takes?|requires?|needs?)\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                        r'completed in\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                        r'(?:typically|usually|generally|normally)\s*(?:takes?|lasts?|runs?)\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                        r'time\s*to\s*(?:complete|finish)\s*(?:is|:)?\s*(?:about|approximately|around)?\s*(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                        r'(\d+)[\s-]*(?:to|-|–)?\s*(\d+)?\s*(?:min|mins|minutes|minute)',
+                        r'(\d+)\s*(?:min|mins|minutes|minute)',
+                    ]
+
+                    # Process duration matches
+                    def process_duration_match(match):
+                        if match.group(2) and match.group(2).isdigit():
+                            # If there's a range, use the maximum value
+                            return f"{match.group(2)} minutes"
+                        else:
+                            return f"{match.group(1)} minutes"
+
+                    # Try each pattern on the description
+                    for pattern in duration_patterns:
+                        duration_match = re.search(pattern, description)
+                        if duration_match:
+                            duration = process_duration_match(duration_match)
+                            break
+
+                    # If still unknown and it's a known test type, assign default durations
+                    if duration == "Unknown":
+                        product_lower = product_name.lower()
+                        if any(word in product_lower for word in ['cognitive', 'numerical', 'verbal', 'reasoning']):
+                            duration = "20 minutes"
+                        elif any(word in product_lower for word in ['personality', 'behavioral']):
+                            duration = "25 minutes"
+                        elif any(word in product_lower for word in ['situational', 'judgment']):
+                            duration = "30 minutes"
+                        elif any(word in product_lower for word in ['coding', 'programming', 'technical']):
+                            duration = "45 minutes"
+
+                    new_products.append({
+                        "Test Name": product_name,
+                        "Remote Testing": remote_testing,
+                        "Adaptive/IRT": adaptive,
+                        "Test Type": test_type,
+                        "Link": product_url,
+                        "Duration": duration
+                    })
+
+                    seen_names.add(product_name)
+
+            # Be nice to the server
+            time.sleep(1)
+
+        except Exception as e:
+            print(f"Error scraping additional page {url}: {e}")
+
+    # Add new products to existing products
+    if new_products:
+        print(f"Found {len(new_products)} additional products")
+        existing_products.extend(new_products)
+
+        # Update CSV with the new products
+        with open('utils\data.csv', 'w', newline='', encoding='utf-8') as csvfile:
+            fieldnames = ["Test Name", "Remote Testing (Yes/No)", "Adaptive/IRT (Yes/No)", "Test Type", "Link", "Duration"]
+            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+
+            writer.writeheader()
+            for product in existing_products:
+                writer.writerow({
+                    "Test Name": product["Test Name"],
+                    "Remote Testing (Yes/No)": product["Remote Testing"],
+                    "Adaptive/IRT (Yes/No)": product["Adaptive/IRT"],
+                    "Test Type": product["Test Type"],
+                    "Link": product["Link"],
+                    "Duration": product["Duration"]
+                })
+
+        print(f"Updated data.csv with {len(existing_products)} total products")
+
+if __name__ == "__main__":
+    scrape_shl_products()

utils/validators.py

@@ -0,0 +1,13 @@
+import re
+
+def url(value):
+
+    pattern = re.compile(
+        r'^(?:http|ftp)s?://'  # http:// or https://
+        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # domain...
+        r'localhost|'  # localhost...
+        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'  # ...or ipv4
+        r'(?::\d+)?'  # optional port
+        r'(?:/?|[/?]\S+)$', re.IGNORECASE)
+    
+    return bool(pattern.match(value))