revert changes
Browse files- app.py +4 -9
- ticker_detail.py +47 -60
app.py
CHANGED
|
@@ -630,15 +630,10 @@ def get_all_etf():
|
|
| 630 |
return etf.getAllEtf()
|
| 631 |
|
| 632 |
@app.get("/ticker/{symbol}")
|
| 633 |
-
|
| 634 |
-
|
| 635 |
-
|
| 636 |
-
|
| 637 |
-
raise HTTPException(status_code=404, detail=f"Symbol {symbol} not found or page error")
|
| 638 |
-
except Exception as e:
|
| 639 |
-
raise HTTPException(status_code=500, detail=str(e))
|
| 640 |
-
|
| 641 |
-
ticker = build_ticker(raw_data, symbol)
|
| 642 |
return ticker
|
| 643 |
|
| 644 |
@app.get("/indices")
|
|
|
|
| 630 |
return etf.getAllEtf()
|
| 631 |
|
| 632 |
@app.get("/ticker/{symbol}")
|
| 633 |
+
def get_ticker_detail(symbol:str):
|
| 634 |
+
scraper = PSXScraper(symbol)
|
| 635 |
+
result = scraper.scrape()
|
| 636 |
+
ticker = build_ticker(result, symbol)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 637 |
return ticker
|
| 638 |
|
| 639 |
@app.get("/indices")
|
ticker_detail.py
CHANGED
|
@@ -1,103 +1,90 @@
|
|
| 1 |
import requests
|
| 2 |
from bs4 import BeautifulSoup
|
| 3 |
-
from datetime import datetime
|
| 4 |
from pydantic import BaseModel
|
| 5 |
from models import TickerData,Ticker,get_market_status
|
| 6 |
import re
|
| 7 |
import httpx
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
from typing import Dict, Optional
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
def ttl_cache(seconds: int):
|
| 15 |
-
def decorator(func):
|
| 16 |
-
cache = {}
|
| 17 |
-
@wraps(func)
|
| 18 |
-
async def wrapper(*args, **kwargs):
|
| 19 |
-
key = (args, tuple(sorted(kwargs.items())))
|
| 20 |
-
now = datetime.utcnow()
|
| 21 |
-
if key in cache:
|
| 22 |
-
result, timestamp = cache[key]
|
| 23 |
-
if now - timestamp < timedelta(seconds=seconds):
|
| 24 |
-
return result
|
| 25 |
-
result = await func(*args, **kwargs)
|
| 26 |
-
cache[key] = (result, now)
|
| 27 |
-
return result
|
| 28 |
-
return wrapper
|
| 29 |
-
return decorator
|
| 30 |
class PSXScraper:
|
| 31 |
|
| 32 |
BASE_URL = "https://dps.psx.com.pk/company/{}"
|
| 33 |
|
| 34 |
-
def __init__(self, symbol
|
| 35 |
self.symbol = symbol.upper()
|
| 36 |
self.url = self.BASE_URL.format(self.symbol)
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
|
| 39 |
# ---------------------------
|
| 40 |
# Fetch Page
|
| 41 |
# ---------------------------
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
return BeautifulSoup(resp.text, "lxml")
|
| 47 |
-
|
| 48 |
-
async def scrape(self, client: httpx.AsyncClient) -> Dict:
|
| 49 |
-
soup = await self.fetch(client)
|
| 50 |
-
# parse_quote_summary and parse_reg_panel are synchronous helpers
|
| 51 |
-
data = {}
|
| 52 |
-
self._parse_quote_summary(soup, data)
|
| 53 |
-
self._parse_reg_panel(soup, data)
|
| 54 |
-
return data
|
| 55 |
|
| 56 |
# ---------------------------
|
| 57 |
# Parse Top Price Section
|
| 58 |
# ---------------------------
|
| 59 |
-
def
|
| 60 |
-
quote = soup.select_one(".quote__price")
|
| 61 |
if not quote:
|
| 62 |
return
|
|
|
|
| 63 |
price = quote.select_one(".quote__close")
|
| 64 |
change = quote.select_one(".change__value")
|
| 65 |
change_pct = quote.select_one(".change__percent")
|
|
|
|
| 66 |
if price:
|
| 67 |
-
data["price"] = price.get_text(strip=True)
|
|
|
|
| 68 |
if change:
|
| 69 |
-
data["change"] = change.get_text(strip=True)
|
|
|
|
| 70 |
if change_pct:
|
| 71 |
-
data["change_percent"] = change_pct.get_text(strip=True)
|
| 72 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
|
| 74 |
-
def _parse_reg_panel(self, soup: BeautifulSoup, data: Dict):
|
| 75 |
-
reg_panel = soup.select_one('div.tabs__panel[data-name="REG"]')
|
| 76 |
if not reg_panel:
|
| 77 |
return
|
| 78 |
-
|
|
|
|
|
|
|
|
|
|
| 79 |
label_el = row.select_one(".stats_label")
|
| 80 |
value_el = row.select_one(".stats_value")
|
|
|
|
| 81 |
if not label_el or not value_el:
|
| 82 |
continue
|
|
|
|
| 83 |
key = normalize_key(label_el.get_text(strip=True))
|
| 84 |
-
|
|
|
|
|
|
|
|
|
|
| 85 |
|
| 86 |
# ---------------------------
|
| 87 |
# Public Method
|
| 88 |
# ---------------------------
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
|
| 95 |
|
| 96 |
-
@ttl_cache(seconds=10) # adjust based on how fresh you need the data
|
| 97 |
-
async def get_ticker_data(symbol: str) -> Dict:
|
| 98 |
-
async with httpx.AsyncClient(timeout=10.0) as client:
|
| 99 |
-
scraper = PSXScraper(symbol)
|
| 100 |
-
return await scraper.scrape(client)
|
| 101 |
|
| 102 |
def normalize_key(label: str) -> str:
|
| 103 |
"""
|
|
@@ -177,6 +164,7 @@ def map_to_ticker_data(raw: dict, symbol: str) -> TickerData:
|
|
| 177 |
year_1_change=to_float(raw.get("1_year_change")),
|
| 178 |
ytd_change=to_float(raw.get("ytd_change")),
|
| 179 |
|
|
|
|
| 180 |
)
|
| 181 |
|
| 182 |
def build_ticker(raw_data: dict, symbol: str) -> Ticker:
|
|
@@ -191,9 +179,8 @@ def build_ticker(raw_data: dict, symbol: str) -> Ticker:
|
|
| 191 |
# Example Usage
|
| 192 |
# ---------------------------
|
| 193 |
if __name__ == "__main__":
|
| 194 |
-
scraper = PSXScraper("
|
| 195 |
result = scraper.scrape()
|
| 196 |
|
| 197 |
-
ticker = build_ticker(result, "
|
| 198 |
-
|
| 199 |
print(ticker.model_dump())
|
|
|
|
| 1 |
import requests
|
| 2 |
from bs4 import BeautifulSoup
|
| 3 |
+
from datetime import datetime
|
| 4 |
from pydantic import BaseModel
|
| 5 |
from models import TickerData,Ticker,get_market_status
|
| 6 |
import re
|
| 7 |
import httpx
|
| 8 |
+
|
| 9 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
class PSXScraper:
|
| 11 |
|
| 12 |
BASE_URL = "https://dps.psx.com.pk/company/{}"
|
| 13 |
|
| 14 |
+
def __init__(self, symbol):
|
| 15 |
self.symbol = symbol.upper()
|
| 16 |
self.url = self.BASE_URL.format(self.symbol)
|
| 17 |
+
self.headers = {
|
| 18 |
+
"User-Agent": "Mozilla/5.0"
|
| 19 |
+
}
|
| 20 |
+
self.soup = None
|
| 21 |
+
self.data = {}
|
| 22 |
|
| 23 |
# ---------------------------
|
| 24 |
# Fetch Page
|
| 25 |
# ---------------------------
|
| 26 |
+
def fetch(self):
|
| 27 |
+
response = requests.get(self.url, headers=self.headers)
|
| 28 |
+
response.raise_for_status()
|
| 29 |
+
self.soup = BeautifulSoup(response.text, "lxml")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
# ---------------------------
|
| 32 |
# Parse Top Price Section
|
| 33 |
# ---------------------------
|
| 34 |
+
def parse_quote_summary(self):
|
| 35 |
+
quote = self.soup.select_one(".quote__price")
|
| 36 |
if not quote:
|
| 37 |
return
|
| 38 |
+
|
| 39 |
price = quote.select_one(".quote__close")
|
| 40 |
change = quote.select_one(".change__value")
|
| 41 |
change_pct = quote.select_one(".change__percent")
|
| 42 |
+
|
| 43 |
if price:
|
| 44 |
+
self.data["price"] = price.get_text(strip=True)
|
| 45 |
+
|
| 46 |
if change:
|
| 47 |
+
self.data["change"] = change.get_text(strip=True)
|
| 48 |
+
|
| 49 |
if change_pct:
|
| 50 |
+
self.data["change_percent"] = change_pct.get_text(strip=True)
|
| 51 |
|
| 52 |
+
# ---------------------------
|
| 53 |
+
# Parse REG Panel Stats
|
| 54 |
+
# ---------------------------
|
| 55 |
+
def parse_reg_panel(self):
|
| 56 |
+
reg_panel = self.soup.select_one(
|
| 57 |
+
'div.tabs__panel[data-name="REG"]'
|
| 58 |
+
)
|
| 59 |
|
|
|
|
|
|
|
| 60 |
if not reg_panel:
|
| 61 |
return
|
| 62 |
+
|
| 63 |
+
rows = reg_panel.select(".stats_item")
|
| 64 |
+
|
| 65 |
+
for row in rows:
|
| 66 |
label_el = row.select_one(".stats_label")
|
| 67 |
value_el = row.select_one(".stats_value")
|
| 68 |
+
|
| 69 |
if not label_el or not value_el:
|
| 70 |
continue
|
| 71 |
+
|
| 72 |
key = normalize_key(label_el.get_text(strip=True))
|
| 73 |
+
|
| 74 |
+
value = value_el.get_text(" ", strip=True)
|
| 75 |
+
|
| 76 |
+
self.data[key] = value
|
| 77 |
|
| 78 |
# ---------------------------
|
| 79 |
# Public Method
|
| 80 |
# ---------------------------
|
| 81 |
+
def scrape(self):
|
| 82 |
+
self.fetch()
|
| 83 |
+
self.parse_quote_summary()
|
| 84 |
+
self.parse_reg_panel()
|
| 85 |
+
return self.data
|
| 86 |
|
| 87 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
def normalize_key(label: str) -> str:
|
| 90 |
"""
|
|
|
|
| 164 |
year_1_change=to_float(raw.get("1_year_change")),
|
| 165 |
ytd_change=to_float(raw.get("ytd_change")),
|
| 166 |
|
| 167 |
+
|
| 168 |
)
|
| 169 |
|
| 170 |
def build_ticker(raw_data: dict, symbol: str) -> Ticker:
|
|
|
|
| 179 |
# Example Usage
|
| 180 |
# ---------------------------
|
| 181 |
if __name__ == "__main__":
|
| 182 |
+
scraper = PSXScraper("NITGETF")
|
| 183 |
result = scraper.scrape()
|
| 184 |
|
| 185 |
+
ticker = build_ticker(result, "NITGETF")
|
|
|
|
| 186 |
print(ticker.model_dump())
|