app.py

from fastapi import FastAPI,HTTPException
from bs4 import BeautifulSoup
import time
import requests
from fastapi.responses import HTMLResponse
from starlette.responses import FileResponse 
from index_data import IndexEnum,INDEX_MAP
from typing import Dict, List
from pydantic import BaseModel
from models import PsxMarketResponse,PsxStock

app = FastAPI(
    title="PSX web scraper",
    docs_url="/",
    description="Scrape data from PSX website"
)


def simple_dividend_extraction():
    """
    Simple function to extract dividend data
    """
    r = requests.get('https://www.psx.com.pk/psx/announcement/financial-announcements')
    print("request done")
    
    
    soup = BeautifulSoup(r.text, 'html.parser')
    table = soup.find('table')
    _table = soup.select('list')
    print(_table)
    print(table)
    
    results = []
    
    if table:
        print("in table")
        
        #rows = table.find('tbody').findChildren('tr')
        rows = table.find_all('tr')[1:]
        print(f"Number of rows found: {len(rows)}")
        
        for row in rows:
            if not hasattr(row, 'find_all'):
                print("no data")
                continue
            print(f"\nRow type: {type(row)}")
            print(f"Row contents: {row}")
            print("in row")
            cells = row.find_all('td')
            
            if len(cells) >= 6:
                company_name = cells[0].get_text(strip=True)
                dividend_amount = cells[3].get_text(strip=True) or "No dividend"
                dividend_date = cells[8].get_text(strip=True) or "No date"
                board_meeting = cells[7].get_text(strip=True) or "No meeting"
                eps = cells[6].get_text(strip=True) or "No eps"
                profit_loss_before_tax = cells[4].get_text(strip=True) or "No profit/loss"
                profit_loss_after_tax = cells[5].get_text(strip=True) or "No profit/loss"
                year_ended = cells[2].get_text(strip=True) or "No profit/loss"
                
                results.append({
                    'Company': company_name,
                    'Dividend': dividend_amount,
                    'Date': dividend_date,
                    'BoardMeeting':board_meeting,
                    "Eps":eps,
                    'profitLossBeforeTax':profit_loss_before_tax,
                    'profitLossAfterTax':profit_loss_after_tax,
                    "yearEnded":year_ended
                })
    
    return results


def get_active_gainers() -> PsxMarketResponse:
    r = requests.get('https://www.psx.com.pk/market-summary/')
    print("request done")
    
    """Extract all market data from the HTML tables"""
    
    # Parse the HTML
    soup = BeautifulSoup(r.text, 'html.parser')
    
    
    main_div = soup.find('div', {'id': 'marketmainboard'})
    
    if not main_div:
        print("Main div not found")
        return {}
    
    all_data = {}
    
    # Find all tables directly (not nested)
    tables = main_div.find_all('table')
    print(f"Found {len(tables)} tables")
    
    for i, table in enumerate(tables):
        # Get the sector/category name from the table header
        header = table.find('th')
        if header:
            h4_tag = header.find('h4')
            if h4_tag:
                sector_name = h4_tag.text.strip()
            else:
                sector_name = header.text.strip()
        else:
            sector_name = f"Sector_{i+1}"
        
        # Skip if it's not a real sector (too short or just column headers)
        if len(sector_name) < 5 or sector_name in ['SCRIP', 'LDCP', 'SYMBOL']:
            continue
            
        print(f"\nProcessing sector {i+1}: {sector_name}")
        
        # Find the tbody - using find() not find_all()
        tbody = table.find('tbody')
        if not tbody:
            print(f"  No tbody found, trying direct rows...")
            # Try to get rows directly from table
            rows = table.find_all('tr')
        else:
            rows = tbody.find_all('tr')
        
        print(f"  Found {len(rows)} rows")
        
        sector_data = []
        row_count = 0
        
        # Process rows
        for row_idx, row in enumerate(rows):
            # Skip header rows
            row_text = row.get_text().lower()
            if any(x in row_text for x in ['scrip', 'ldcp', 'open', 'high', 'low', 'current', 'change', 'volume']):
                print(f"  Skipping header row: {row_text[:50]}...")
                continue
            
            columns = row.find_all('td')
            
            # Check if this is a data row (should have 8 columns)
            if len(columns) >= 8:
                row_count += 1
                try:
                    # Extract script name and code
                    script_name = ""
                    script_code = ""
                    
                    first_col = columns[0]
                    
                    # Try to find dataportal class for script code
                    dataportal_cell = first_col.find(class_='dataportal')
                    if dataportal_cell:
                        if dataportal_cell.has_attr('data-srip'):
                            script_code = dataportal_cell['data-srip']
                        script_name = dataportal_cell.get_text(strip=True)
                    else:
                        script_name = first_col.get_text(strip=True)
                    
                    # Clean numeric values
                    def clean_numeric(value):
                        # Remove commas and non-numeric chars except decimal and minus
                        cleaned = ''.join(c for c in str(value) if c.isdigit() or c in '.-')
                        return cleaned if cleaned else "0"
                    
                    # Extract change value (might have span inside)
                    change_cell = columns[6]
                    change_text = change_cell.get_text(strip=True)
                    
                    # Determine trend
                    trend = 'increase'
                    if 'red-text-td' in row.get('class', []):
                        trend = 'decrease'
                    elif 'green-text-td' in row.get('class', []):
                        trend = 'increase'
                    
                    stock = PsxStock(
                        script_code=script_code,
                        script_name=script_name,
                        ldcp=clean_numeric(columns[1].get_text(strip=True)),
                        open=clean_numeric(columns[2].get_text(strip=True)),
                        high=clean_numeric(columns[3].get_text(strip=True)),
                        low=clean_numeric(columns[4].get_text(strip=True)),
                        current=clean_numeric(columns[5].get_text(strip=True)),
                        change=clean_numeric(columns[6].get_text(strip=True)),
                        volume=clean_numeric(columns[7].get_text(strip=True)),
                        trend=trend
                    )
                    
                    sector_data.append(stock)
                    
                except Exception as e:
                    print(f"  Error processing row {row_idx}: {e}")
                    print(f"  Row columns: {[col.get_text(strip=True)[:20] for col in columns]}")
                    continue
        
        if sector_data:
            all_data[sector_name] = sector_data
            print(f"  Successfully extracted {len(sector_data)} records")
        else:
            print(f"  No data extracted from {sector_name}")
    
    return PsxMarketResponse(sectors=all_data)


@app.get("/hello")
def greet_json():
    return {"Hello": "World!"}


@app.get("/dividend_history")
def get_dividend():
    return simple_dividend_extraction()

@app.get("/PrivacyPolicy")
def get_privacy_policy():
   
    return FileResponse('text.html')

@app.get("/portfolio")
def get_portfolio():
    return FileResponse('portfolio.html')



@app.get("/index/{index_name}")
def get_index_companies(index_name: IndexEnum):
    companies = INDEX_MAP.get(index_name)

    if companies is None:
        raise HTTPException(status_code=404, detail="Index not found")

    return [company.to_dict() for company in companies]



@app.get("/gainer_loosers")
def get_gainers_loosers():
    
    return get_active_gainers()