initial commit

.gitignore

@@ -0,0 +1,32 @@
+.vercel
+*.pyc
+venv/
+__pycache__/
+# Ignore Python bytecode files
+*.pyo
+# Ignore Python cache directories
+__pycache__/
+# Ignore virtual environment directories
+.env
+# Ignore Jupyter Notebook checkpoints
+.ipynb_checkpoints/
+# Ignore log files
+*.log
+# Ignore coverage reports
+.coverage
+# Ignore build directories
+build/
+dist/
+# Ignore package distribution files
+*.egg-info/
+# Ignore IDE/editor specific files
+.vscode/
+.idea/
+# Ignore system files
+.DS_Store
+# Ignore environment variable files
+.env.local
+# Ignore Python egg files
+*.egg
+*.sql*
+bash.exe.stackdump

App/requirements.txt

@@ -0,0 +1,45 @@
+aiosqlite==0.17.0
+anyio==3.7.1
+asgiref==3.8.1
+asynctest==0.13.0
+bcrypt==4.0.1
+beautifulsoup4==4.13.4
+certifi==2025.1.31
+cffi==1.17.1
+charset-normalizer==3.4.1
+click==8.1.8
+colorama==0.4.6
+curl_cffi==0.11.1
+dnspython==2.7.0
+email_validator==2.2.0
+fastapi==0.95.2
+h11==0.12.0
+httpcore==0.15.0
+httpx==0.23.0
+idna==3.10
+iniconfig==2.1.0
+iso8601==1.1.0
+numpy==1.24.3
+packaging==24.2
+pandas==2.2.3
+passlib==1.7.4
+pluggy==1.5.0
+pycparser==2.22
+pydantic==1.10.7
+pypika-tortoise==0.1.6
+pytest==7.3.1
+pytest-asyncio==0.21.0
+python-dateutil==2.9.0.post0
+python-multipart==0.0.5
+pytz==2025.2
+requests==2.31.0
+rfc3986==1.5.0
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.7
+starlette==0.27.0
+tortoise-orm==0.19.3
+typing_extensions==4.13.1
+tzdata==2025.2
+urllib3==2.3.0
+uvicorn==0.22.0

App/routers/bonds/__init__.py

@@ -0,0 +1,3 @@
+from .routes import router
+
+__all__ = ["router"]

App/routers/bonds/models.py

@@ -0,0 +1,44 @@
+from tortoise import fields, models
+from tortoise.contrib.pydantic import pydantic_model_creator
+from tortoise.contrib.pydantic.creator import pydantic_queryset_creator
+from tortoise.queryset import QuerySet
+
+class Bond(models.Model):
+    id = fields.IntField(pk=True)
+    instrument_type = fields.CharField(max_length=50)
+    auction_number = fields.IntField()
+    auction_date = fields.DateField()
+    maturity_years = fields.CharField(max_length=10)
+    maturity_date = fields.DateField()
+    effective_date = fields.DateField()
+    dtm = fields.IntField()
+    bond_auction_number = fields.IntField()
+    holding_number = fields.IntField()
+    face_value = fields.BigIntField()
+    price_per_100 = fields.FloatField()
+    coupon_rate = fields.FloatField()
+    isin = fields.CharField(max_length=12, unique=True)
+
+    @staticmethod
+    async def get_list(data):
+
+        if type(data) == QuerySet:
+            parser=pydantic_queryset_creator(Bond)
+            return await parser.from_queryset(data)
+
+        if type(data) == type([Bond]):
+            return [ await i.to_dict() for i in data]
+
+
+    async def to_dict(self):
+        if type(self) == Bond:
+            parser=pydantic_model_creator(Bond)
+            return await parser.from_tortoise_orm(self)
+
+
+        
+        
+
+    class Meta:
+        table = "bonds"
+        unique_together = ("auction_number", "auction_date")

App/routers/bonds/routes.py

@@ -0,0 +1,126 @@
+from fastapi import APIRouter, BackgroundTasks, HTTPException
+from tortoise.contrib.pydantic.creator import pydantic_queryset_creator
+from tortoise.transactions import in_transaction
+from App.routers.bonds.models import Bond # Adjust import path
+from App.routers.tasks.models import ImportTask # Adjust import path
+from App.routers.bonds.schemas import BondCreate, BondResponse # Adjust import path
+from App.routers.bonds.utils import BondDataScraper # Adjust import path
+from App.schemas import ResponseModel # Assuming you have this general response model
+from typing import List
+
+
+router = APIRouter(prefix="/bonds", tags=["Bonds"])
+
+# --- CRUD for Bond (example, you might have these elsewhere) ---
+@router.post("/", response_model=ResponseModel)
+async def create_bond_entry(payload: BondCreate):
+    # Check for existing bond using ISIN or combination of auction_number, auction_date, holding_number
+    existing_bond = None
+    if payload.isin:
+        existing_bond = await Bond.get_or_none(isin=payload.isin)
+    
+    if not existing_bond:
+         existing_bond = await Bond.get_or_none(
+            auction_number=payload.auction_number,
+            auction_date=payload.auction_date,
+            holding_number=payload.holding_number # or bond_auction_number if that's more unique
+        )
+
+    if existing_bond:
+        # Update existing bond
+        await Bond.filter(id=existing_bond.id).update(**payload.dict(exclude_unset=True))
+        bond = await Bond.get(id=existing_bond.id)
+        message = "Bond updated successfully"
+    else:
+        # Create new bond
+        bond = await Bond.create(**payload.dict())
+        message = "Bond created successfully"
+        
+    return ResponseModel(success=True, message=message, data=await BondResponse.from_tortoise_orm(bond))
+
+@router.get("/", response_model=ResponseModel)
+async def list_bonds_entries():
+    
+    _bonds = await Bond.all()
+    print(_bonds)
+    bonds= await Bond.get_list(_bonds)
+    print(bonds)
+    
+    return ResponseModel(success=True, message="Bonds retrieved successfully", data={"bonds": bonds})
+
+# --- Import Task ---
+async def run_bond_import_task(task_id: int):
+    await ImportTask.filter(id=task_id).update(status="running")
+    scraper = BondDataScraper()
+    created_count = 0
+    updated_count = 0
+    failed_count = 0
+    processed_isins = set()
+
+    try:
+        async for bond_data in scraper.scrape_all_bond_data():
+            if not bond_data:
+                failed_count += 1
+                continue
+            
+            if bond_data.isin and bond_data.isin in processed_isins:
+                print(f"Skipping duplicate ISIN in current scrape: {bond_data.isin}")
+                continue
+
+            async with in_transaction(): # Ensure atomicity for each bond
+                try:
+                    # Use ISIN as primary unique key if available, otherwise fallback
+                    existing_bond = None
+                    if bond_data.isin:
+                        existing_bond = await Bond.get_or_none(isin=bond_data.isin)
+                    
+                    if not existing_bond: # Fallback check
+                        existing_bond = await Bond.get_or_none(
+                            auction_number=bond_data.auction_number,
+                            auction_date=bond_data.auction_date,
+                            # Add holding_number or bond_auction_number if they form part of a unique key
+                            holding_number=bond_data.holding_number 
+                        )
+
+                    if existing_bond:
+                        await Bond.filter(id=existing_bond.id).update(**bond_data.dict(exclude_unset=True))
+                        updated_count += 1
+                        print(f"Updated bond: ISIN {bond_data.isin}, AuNo {bond_data.auction_number}")
+                    else:
+                        await Bond.create(**bond_data.dict())
+                        created_count += 1
+                        print(f"Created bond: ISIN {bond_data.isin}, AuNo {bond_data.auction_number}")
+                    
+                    if bond_data.isin:
+                        processed_isins.add(bond_data.isin)
+
+                except Exception as db_e:
+                    failed_count += 1
+                    print(f"Database error for bond au_no {bond_data.auction_number}: {db_e}")
+        
+        summary = {
+            "created": created_count,
+            "updated": updated_count,
+            "failed_during_processing": failed_count,
+            "message": "Bond import process finished."
+        }
+        await ImportTask.filter(id=task_id).update(status="completed", details=summary)
+        print(f"Bond import task {task_id} completed. Summary: {summary}")
+
+    except Exception as e:
+        print(f"Fatal error in bond import task {task_id}: {e}")
+        await ImportTask.filter(id=task_id).update(status="failed", details={"error": str(e)})
+
+
+@router.post("/import-bonds", response_model=ResponseModel)
+async def trigger_bond_import(background_tasks: BackgroundTasks):
+    task = await ImportTask.create(task_type="bond_import", status="pending")
+    background_tasks.add_task(run_bond_import_task, task.id)
+    return ResponseModel(success=True, message="Bond import task started.", data={"task_id": task.id})
+
+@router.get("/import-status/{task_id}", response_model=ResponseModel)
+async def get_import_status(task_id: int):
+    task = await ImportTask.get_or_none(id=task_id)
+    if not task:
+        raise HTTPException(status_code=404, detail="Import task not found")
+    return ResponseModel(success=True, message="Task status retrieved", data=task)

App/routers/bonds/schemas.py

@@ -0,0 +1,24 @@
+from pydantic import BaseModel
+from typing import Optional
+from datetime import date
+
+class BondCreate(BaseModel):
+    instrument_type: str
+    auction_number: int
+    auction_date: date
+    maturity_years: Optional[str]
+    maturity_date: date
+    effective_date: date
+    dtm: int
+    bond_auction_number: int
+    holding_number: int
+    face_value: int
+    price_per_100: float
+    coupon_rate: Optional[float]
+    isin:str
+
+class BondResponse(BondCreate):
+    id: int
+    
+    class Config:
+        orm_mode = True

App/routers/bonds/service.py

@@ -0,0 +1,23 @@
+from datetime import datetime, timedelta
+from typing import List
+
+def calculate_coupon_dates(start_date: datetime, maturity_date: datetime, interval_months: int = 6) -> List[datetime]:
+    """Calculate coupon payment dates for a bond.
+    
+    Args:
+        start_date: The effective date of the bond
+        maturity_date: The maturity date of the bond
+        interval_months: Interval between coupon payments in months (default 6)
+        
+    Returns:
+        List of coupon payment dates
+    """
+    dates = []
+    current_date = start_date
+    
+    while current_date < maturity_date:
+        if current_date.weekday() < 5:  # Skip weekends
+            dates.append(current_date)
+        current_date += timedelta(days=interval_months * 30.44)  # Approximate months
+    
+    return dates

App/routers/bonds/utils.py

@@ -0,0 +1,258 @@
+import asyncio
+from curl_cffi.requests import AsyncSession,RequestsError
+from bs4 import BeautifulSoup
+from App.routers.bonds.schemas import BondCreate # Adjust import path
+# from .bond_utils import parse_bond_title_details, parse_date_flexible, get_summary_item_value # If in separate file
+from typing import List, Dict, Any, Optional, AsyncGenerator
+
+import re
+from datetime import datetime as dt
+from typing import Tuple, Optional, List, Dict, Any
+
+def parse_bond_title_details(title_str: str) -> Tuple[Optional[float], Optional[str], str, Optional[int]]:
+    coupon_rate_val = None
+    maturity_years_val = None
+    instrument_type_val = "TREASURY BOND" 
+    issue_number_val = None
+
+    if not title_str:
+        return coupon_rate_val, maturity_years_val, instrument_type_val, issue_number_val
+
+    # 1. Coupon Rate
+    coupon_match = re.search(r'(\d+\.?\d*)%', title_str)
+    if coupon_match:
+        coupon_rate_val = float(coupon_match.group(1))
+        remaining_after_coupon = title_str.split(coupon_match.group(0), 1)[-1].strip()
+    else:
+        remaining_after_coupon = title_str
+
+    # 2. Maturity Years
+    maturity_match = re.search(r'(\d+)-YEAR', remaining_after_coupon, re.IGNORECASE)
+    if maturity_match:
+        maturity_years_val = f"{maturity_match.group(1)}-YEAR"
+        remaining_after_year = remaining_after_coupon.split(maturity_match.group(0), 1)[-1].strip()
+    else:
+        remaining_after_year = remaining_after_coupon
+    
+    # 3. Instrument Type (base part)
+    # Remove "NUMBER ..." and "ISSUE ..." parts for cleaner type detection
+    cleaner_remaining = re.split(r'\s+NUMBER\s+\d+', remaining_after_year, flags=re.IGNORECASE)[0]
+    cleaner_remaining = re.split(r'\s+ISSUE\s+\d+', cleaner_remaining, flags=re.IGNORECASE)[0].strip()
+
+    if cleaner_remaining:
+        instrument_type_val = cleaner_remaining
+        if maturity_years_val and maturity_years_val not in instrument_type_val : # Prepend year if not already part of it
+             instrument_type_val = f"{maturity_years_val} {instrument_type_val}"
+    elif maturity_years_val:
+        instrument_type_val = f"{maturity_years_val} TREASURY BOND"
+    
+    # 4. Issue Number (search in original title for issue)
+    issue_match = re.search(r'ISSUE\s+(\d+)', title_str, re.IGNORECASE)
+    if issue_match:
+        issue_number_val = int(issue_match.group(1))
+    
+    if not instrument_type_val.strip() and title_str.strip(): # Fallback
+        instrument_type_val = "TREASURY BOND"
+        if maturity_years_val:
+            instrument_type_val = f"{maturity_years_val} {instrument_type_val}"
+
+
+    return coupon_rate_val, maturity_years_val, instrument_type_val.strip(), issue_number_val
+
+
+def parse_date_flexible(date_str: str, default=None) -> Optional[dt]:
+    if not date_str:
+        return default
+    # Handle cases like "27-DEC-2012"
+    date_str = date_str.replace("-", " ").title() if len(date_str.split('-')) == 3 else date_str
+    
+    formats_to_try = ["%d %b %Y", "%B %d, %Y", "%d/%m/%Y"]
+    for fmt in formats_to_try:
+        try:
+            return dt.strptime(date_str, fmt).date()
+        except ValueError:
+            continue
+    print(f"Warning: Could not parse date string: {date_str}")
+    return default
+
+def get_summary_item_value(summary_list: List[Dict[str, Any]], item_desc_key: str, default=None) -> Optional[str]:
+    for item in summary_list:
+        if item.get("itemDesc", "").strip().upper() == item_desc_key.upper():
+            value_str = item.get("itemValue")
+            if value_str:
+                return value_str.strip()
+    return default
+
+
+class BondDataScraper:
+    BASE_URL = "https://www.bot.go.tz"
+    TBONDS_URL = f"{BASE_URL}/TBonds"
+    AUCTION_SUMMARY_URL = f"{BASE_URL}/TBonds/AuctionSummaries"
+    IMPERSONATE_PROFILE = "chrome110" # Or another recent Chrome version
+
+    def __init__(self):
+        self.headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+            'Accept-Language': 'en-US,en;q=0.9',
+        }
+        self.ajax_headers = {
+            **self.headers,
+            'Accept': 'application/json, text/javascript, */*; q=0.01', # Adjusted for AJAX
+            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+            'X-Requested-With': 'XMLHttpRequest',
+            'Origin': self.BASE_URL,
+            'Referer': self.TBONDS_URL,
+        }
+
+    async def _fetch_content(self, session: AsyncSession, url: str, method: str = "GET", data: Optional[Dict] = None, is_json: bool = False):
+        try:
+            if method.upper() == "POST":
+                response = await session.post(url, headers=self.ajax_headers if data else self.headers, data=data, impersonate=self.IMPERSONATE_PROFILE, timeout=20)
+            else:
+                response = await session.get(url, headers=self.headers, impersonate=self.IMPERSONATE_PROFILE, timeout=60*5)
+                
+            
+            response.raise_for_status()
+            return  response.json() if is_json else response.text
+        except RequestsError as e: # Updated exception type for curl_cffi
+            print(f"HTTP error fetching {url}: {e}") # curl_cffi errors might not have response.status_code directly
+        except Exception as e:
+            print(f"Unexpected error fetching {url}: {e}")
+        return None
+
+    async def _parse_main_tbonds_page(self, html_content: str) -> List[Dict[str, Any]]:
+        if not html_content:
+            return []
+        
+        soup = BeautifulSoup(html_content, 'html.parser')
+        table = soup.find('table', class_='tbond-table') # Or id="DataTables_Table_0"
+        if not table:
+            print("Main T-Bonds table not found.")
+            return []
+
+        parsed_rows = []
+        tbody = table.find('tbody')
+        if not tbody:
+            print("Tbody not found in main T-Bonds table.")
+            return []
+            
+        for row in tbody.find_all('tr'):
+            cols = row.find_all(['th', 'td']) # First col might be th
+            if len(cols) < 5:
+                continue
+
+            try:
+                # Sn. is cols[0]
+                auction_number_text = cols[1].get_text(strip=True)
+                auction_title = cols[2].get_text(strip=True)
+                auction_date_str = cols[3].get_text(strip=True)
+                
+                view_button = cols[4].find('button', id='showSummaryDetails')
+                if not view_button or not view_button.get('value'):
+                    print(f"Skipping row, view button or value not found for auction: {auction_number_text}")
+                    continue
+                
+                button_value_parts = view_button['value'].split('_')
+                au_no_str = button_value_parts[0]
+                au_days_part = button_value_parts[1] # Usually '1'
+
+                parsed_rows.append({
+                    'table_auction_number_text': auction_number_text, # This is the au_no
+                    'table_auction_title': auction_title,
+                    'table_auction_date_str': auction_date_str,
+                    'au_no': int(au_no_str),
+                    'au_days_part': au_days_part,
+                })
+            except Exception as e:
+                print(f"Error parsing main table row: {e}. Row: {[c.get_text(strip=True) for c in cols]}")
+        return parsed_rows
+
+    async def _fetch_bond_details(self, session: AsyncSession, au_no: int, au_days_part: str) -> Optional[Dict[str, Any]]:
+        payload = {'au_no': str(au_no), 'au_days': au_days_part}
+        return await self._fetch_content(session, self.AUCTION_SUMMARY_URL, method="POST", data=payload, is_json=True)
+
+    def _parse_bond_details_json(self, json_data: Dict[str, Any], initial_data: Dict[str, Any]) -> Optional[BondCreate]:
+        if not json_data or json_data.get("message") != "SUCCESS":
+            print(f"Bond details JSON invalid or not successful for au_no: {initial_data.get('au_no')}")
+            return None
+
+        summary_list = json_data.get("tbondSummary", [])
+        
+        coupon_rate, maturity_years, instrument_type, issue_number = parse_bond_title_details(json_data.get("bondTitle", ""))
+        
+        auction_date_obj = parse_date_flexible(initial_data['table_auction_date_str'])
+        if not auction_date_obj: # Critical, skip if no valid auction date
+             print(f"Critical: Could not parse auction_date for au_no: {initial_data.get('au_no')}")
+             return None
+
+
+        maturity_date_str = get_summary_item_value(summary_list, "REDEMPTION DATE")
+        maturity_date_obj = parse_date_flexible(maturity_date_str)
+        
+        dtm_val = None
+        if maturity_date_obj and auction_date_obj:
+            dtm_val = (maturity_date_obj - auction_date_obj).days
+
+        face_value_str = get_summary_item_value(summary_list, "AMOUNT OFFERED TZS(000,000)")
+        face_value_val = None
+        if face_value_str:
+            try:
+                face_value_val = int(float(face_value_str.replace(",", "")) * 1_000_000)
+            except ValueError:
+                print(f"Could not parse face_value: {face_value_str} for au_no: {initial_data.get('au_no')}")
+
+        price_per_100_str = get_summary_item_value(summary_list, "MINIMUM SUCCESSFUL PRICE / 100") # Or WAP?
+        price_per_100_val = None
+        if price_per_100_str:
+            try:
+                price_per_100_val = float(price_per_100_str)
+            except ValueError:
+                 print(f"Could not parse price_per_100: {price_per_100_str} for au_no: {initial_data.get('au_no')}")
+
+
+        holding_number_str = json_data.get("auctionNumber") # This is the "1" from "AUCTION NUMBER 1 HELD ON..."
+        holding_number_val = int(holding_number_str) if holding_number_str and holding_number_str.isdigit() else None
+
+        return BondCreate(
+            instrument_type=instrument_type if instrument_type else "TREASURY BOND",
+            auction_number=initial_data['au_no'], # This is the main identifier from table
+            auction_date=auction_date_obj,
+            maturity_years=maturity_years,
+            maturity_date=maturity_date_obj,
+            effective_date=auction_date_obj, # Assuming effective date is auction date
+            dtm=dtm_val,
+            bond_auction_number=issue_number, # Issue number from title
+            holding_number=holding_number_val, # From JSON details header auctionNumber
+            face_value=face_value_val,
+            price_per_100=price_per_100_val,
+            coupon_rate=coupon_rate,
+            isin=json_data.get("ISIN")
+        )
+
+    async def scrape_all_bond_data(self) -> AsyncGenerator[BondCreate, None]:
+        async with AsyncSession() as session:
+            # First GET request to establish session cookies if necessary
+            await session.get(self.TBONDS_URL, headers=self.headers, impersonate=self.IMPERSONATE_PROFILE,timeout=60*5)
+
+            main_page_html = await self._fetch_content(session, self.TBONDS_URL, method="GET")
+            print(main_page_html)
+            if not main_page_html:
+                print("Failed to fetch main T-Bonds page.")
+                return
+
+            initial_bond_rows = await self._parse_main_tbonds_page(main_page_html)
+            
+            print(f"Found {len(initial_bond_rows)} initial bond rows from main table.")
+
+            for row_data in initial_bond_rows:
+                print(f"Fetching details for au_no: {row_data['au_no']}...")
+                await asyncio.sleep(0.5) # Small delay to be polite
+                
+                details_json = await self._fetch_bond_details(session, row_data['au_no'], row_data['au_days_part'])
+                if details_json:
+                    bond_create_obj = self._parse_bond_details_json(details_json, row_data)
+                    if bond_create_obj:
+                        yield bond_create_obj
+                else:
+                    print(f"Failed to fetch or parse details for au_no: {row_data['au_no']}")

App/routers/portfolio/models.py

@@ -0,0 +1,183 @@
+ # models.py
+from tortoise import fields, models
+from typing import Optional
+from datetime import datetime
+
+from tortoise.contrib.pydantic.creator import pydantic_model_creator, pydantic_queryset_creator
+from tortoise.queryset import QuerySet
+class Portfolio(models.Model):
+    id = fields.IntField(pk=True)
+    user = fields.ForeignKeyField("models.User", related_name="portfolios")
+    name = fields.CharField(max_length=100)
+    description = fields.TextField(null=True)
+    is_active = fields.BooleanField(default=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_now=True)
+
+    async def to_dict(self):
+        if type(self) == models.Model:
+            parser = pydantic_model_creator(Portfolio)
+            return await parser.from_tortoise_orm(self)
+        if type(self) == QuerySet:
+            parser = pydantic_queryset_creator(Portfolio)
+            return await parser.from_queryset(self)
+
+
+    class Meta:
+        table = "portfolios"
+        unique_together = ("user", "name")  # User can't have duplicate portfolio names
+
+class PortfolioStock(models.Model):
+    id = fields.IntField(pk=True)
+    portfolio = fields.ForeignKeyField("models.Portfolio", related_name="stocks")
+    stock = fields.ForeignKeyField("models.Stock", related_name="portfolio_holdings")
+    quantity = fields.IntField()
+    purchase_price = fields.DecimalField(max_digits=15, decimal_places=2)
+    purchase_date = fields.DateField()
+    notes = fields.TextField(null=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_now=True)
+
+    async def to_dict(self):
+        if type(self) == models.Model:
+            parser = pydantic_model_creator(PortfolioStock)
+            return await parser.from_tortoise_orm(self)
+        if type(self) == QuerySet:
+            parser = pydantic_queryset_creator(PortfolioStock)
+            return await parser.from_queryset(self)
+
+    class Meta:
+        table = "portfolio_stocks"
+
+class PortfolioUTT(models.Model):
+    id = fields.IntField(pk=True)
+    portfolio = fields.ForeignKeyField("models.Portfolio", related_name="utts")
+    utt_fund = fields.ForeignKeyField("models.UTTFund", related_name="portfolio_holdings")
+    units_held = fields.DecimalField(max_digits=15, decimal_places=4)
+    purchase_price = fields.DecimalField(max_digits=15, decimal_places=2)
+    purchase_date = fields.DateField()
+    notes = fields.TextField(null=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_now=True)
+
+    async def to_dict(self):
+        if type(self) == models.Model:
+            parser = pydantic_model_creator(PortfolioUTT)
+            return await parser.from_tortoise_orm(self)
+        if type(self) == QuerySet:
+            parser = pydantic_queryset_creator(PortfolioUTT)
+            return await parser.from_queryset(self)
+
+    class Meta:
+        table = "portfolio_utts"
+
+class PortfolioBond(models.Model):
+    id = fields.IntField(pk=True)
+    portfolio = fields.ForeignKeyField("models.Portfolio", related_name="bonds")
+    bond = fields.ForeignKeyField("models.Bond", related_name="portfolio_holdings")
+    face_value_held = fields.BigIntField()
+    purchase_price = fields.DecimalField(max_digits=15, decimal_places=2)
+    purchase_date = fields.DateField()
+    notes = fields.TextField(null=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_now=True)
+
+    async def to_dict(self):
+        if type(self) == models.Model:
+            parser = pydantic_model_creator(PortfolioBond)
+            return await parser.from_tortoise_orm(self)
+        if type(self) == QuerySet:
+            parser = pydantic_queryset_creator(PortfolioBond)
+            return await parser.from_queryset(self)
+
+    class Meta:
+        table = "portfolio_bonds"
+
+class PortfolioTransaction(models.Model):
+    """Track all portfolio transactions for audit and reporting"""
+    id = fields.IntField(pk=True)
+    portfolio = fields.ForeignKeyField("models.Portfolio", related_name="transactions")
+    transaction_type = fields.CharField(max_length=20)  # BUY, SELL, DIVIDEND, COUPON
+    asset_type = fields.CharField(max_length=10)  # STOCK, BOND, UTT
+    asset_id = fields.IntField()  # Generic reference to stock/bond/utt ID
+    quantity = fields.DecimalField(max_digits=15, decimal_places=4)
+    price = fields.DecimalField(max_digits=15, decimal_places=2)
+    total_amount = fields.DecimalField(max_digits=15, decimal_places=2)
+    transaction_date = fields.DateField()
+    notes = fields.TextField(null=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+
+    @staticmethod
+    async def get_list(data):
+        if type(data) == QuerySet:
+            parser = pydantic_queryset_creator(PortfolioTransaction)
+            return await parser.from_queryset(data)
+
+    async def to_dict(self):
+        if type(self) == models.Model:
+            parser = pydantic_model_creator(PortfolioTransaction)
+            return await parser.from_tortoise_orm(self)
+
+
+    class Meta:
+        table = "portfolio_transactions"
+
+class PortfolioCalendar(models.Model):
+    id = fields.IntField(pk=True)
+    portfolio = fields.ForeignKeyField("models.Portfolio", related_name="calendar_events")
+    event_date = fields.DateField()
+    event_type = fields.CharField(max_length=50)  # COUPON, DIVIDEND, MATURITY, EARNINGS
+    title = fields.CharField(max_length=200)
+    description = fields.TextField(null=True)
+    asset_type = fields.CharField(max_length=10, null=True)  # STOCK, BOND, UTT
+    asset_id = fields.IntField(null=True)
+    estimated_amount = fields.DecimalField(max_digits=15, decimal_places=2, null=True)
+    is_completed = fields.BooleanField(default=False)
+    created_at = fields.DatetimeField(auto_now_add=True)
+
+    @staticmethod
+    async def get_list(data):
+        if type(data) == QuerySet:
+            parser = pydantic_queryset_creator(PortfolioCalendar)
+            return await parser.from_queryset(data)
+
+
+    async def to_dict(self):
+        if type(self) == models.Model:
+            parser = pydantic_model_creator(PortfolioCalendar)
+            return await parser.from_tortoise_orm(self)
+
+
+    class Meta:
+        table = "portfolio_calendar"
+
+class PortfolioSnapshot(models.Model):
+    """Daily snapshots for performance tracking"""
+    id = fields.IntField(pk=True)
+    portfolio = fields.ForeignKeyField("models.Portfolio", related_name="snapshots")
+    snapshot_date = fields.DatetimeField()
+    total_value = fields.DecimalField(max_digits=20, decimal_places=2)
+    stock_value = fields.DecimalField(max_digits=20, decimal_places=2, default=0)
+    bond_value = fields.DecimalField(max_digits=20, decimal_places=2, default=0)
+    utt_value = fields.DecimalField(max_digits=20, decimal_places=2, default=0)
+    cash_value = fields.DecimalField(max_digits=20, decimal_places=2, default=0)
+    total_cost = fields.DecimalField(max_digits=20, decimal_places=2)
+    unrealized_gain_loss = fields.DecimalField(max_digits=20, decimal_places=2)
+    created_at = fields.DatetimeField(auto_now_add=True)
+
+
+    @staticmethod
+    async def get_list(data):
+        if type(data) == QuerySet:
+            parser = pydantic_queryset_creator(PortfolioSnapshot)
+            return await parser.from_queryset(data)
+
+    async def to_dict(self):
+        if type(self) == models.Model:
+            parser = pydantic_model_creator(PortfolioSnapshot)
+            return await parser.from_tortoise_orm(self)
+
+
+    class Meta:
+        table = "portfolio_snapshots"
+        unique_together = ("portfolio", "snapshot_date")

App/routers/portfolio/routes.py

@@ -0,0 +1,1301 @@
+# routes.py
+from fastapi import APIRouter, Depends, HTTPException, Query
+from typing import List, Optional
+from datetime import date
+from App.routers.bonds.models import Bond  # Import Bond model
+from .service import _calculate_bond_coupon_dates  # Import our
+from decimal import Decimal  # Import Decimal for type hints if necessary
+from tortoise.exceptions import DoesNotExist
+from App.routers.utt.models import UTTFundData
+from App.routers.users.utils import get_current_user
+from .models import (
+    Portfolio,
+    PortfolioSnapshot,
+    PortfolioBond,
+    PortfolioCalendar,
+    PortfolioTransaction,
+    PortfolioStock,
+    PortfolioUTT,
+)
+from .schemas import (
+    PortfolioCreate,
+    PortfolioUpdate,
+    PortfolioBase,
+    PortfolioSummary,
+    StockHoldingCreate,
+    StockHoldingUpdate,
+    StockHoldingResponse,
+    UTTHoldingCreate,
+    UTTHoldingUpdate,
+    UTTHoldingResponse,
+    BondHoldingCreate,
+    BondHoldingUpdate,
+    BondHoldingResponse,
+    CalendarEventCreate,
+    CalendarEventResponse,
+    TransactionDetailResponse,
+    PortfolioListResponse,
+    PositionResponse,
+    StockSellSchema,
+    UTTSellSchema,
+    BondSellSchema,
+)
+from .service import PortfolioService
+from App.schemas import ResponseModel, AppException
+from tortoise.contrib.pydantic import pydantic_model_creator, pydantic_queryset_creator
+
+
+from fastapi import BackgroundTasks
+from .service import PortfolioService  # Ensure service is imported
+from App.routers.tasks.models import ImportTask
+from tortoise.expressions import Q  # For querying JSON fields
+from datetime import date
+from datetime import date, datetime, timedelta
+from App.routers.stocks.models import Dividend, Stock, StockPriceData
+from decimal import Decimal
+from .schemas import CalendarEventResponse  # Import our new schema
+from .models import Portfolio, PortfolioStock, PortfolioBond
+from App.routers.utt.models import UTTFund
+from App.routers.bonds.models import Bond
+
+Portfolio_Pydantic = pydantic_model_creator(Portfolio, name="Portfolio")
+PortfolioStock_Pydantic = pydantic_model_creator(PortfolioStock, name="PortfolioStock")
+PortfolioUTT_Pydantic = pydantic_model_creator(PortfolioUTT, name="PortfolioUTT")
+PortfolioBond_Pydantic = pydantic_model_creator(PortfolioBond, name="PortfolioBond")
+PortfolioTransaction_Pydantic = pydantic_model_creator(
+    PortfolioTransaction, name="PortfolioTransaction"
+)
+PortfolioCalendar_Pydantic = pydantic_model_creator(
+    PortfolioCalendar, name="PortfolioCalendar"
+)
+
+Portfolio_Pydantic_List = pydantic_queryset_creator(Portfolio, name="PortfolioList")
+# Not strictly needed if manually converting list items, but good for consistency
+# PortfolioStock_Pydantic_List = pydantic_queryset_creator(PortfolioStock, name="PortfolioStockList")
+# PortfolioUTT_Pydantic_List = pydantic_queryset_creator(PortfolioUTT, name="PortfolioUTTList")
+# PortfolioBond_Pydantic_List = pydantic_queryset_creator(PortfolioBond, name="PortfolioBondList")
+# PortfolioTransaction_Pydantic_List = pydantic_queryset_creator(PortfolioTransaction, name="PortfolioTransactionList")
+# PortfolioCalendar_Pydantic_List = pydantic_queryset_creator(PortfolioCalendar, name="PortfolioCalendarList")
+PortfolioSnapshotPydantic = pydantic_model_creator(
+    PortfolioSnapshot, name="PortfolioSnapshotResponse"
+)  # Renamed for clarity
+
+router = APIRouter(prefix="/portfolios", tags=["portfolios"])
+
+# Portfolio Management Routes
+
+
+@router.get("/", response_model=ResponseModel)
+async def get_user_portfolios(
+    include_inactive: bool = Query(False), current_user=Depends(get_current_user)
+):
+    try:
+        portfolios = await PortfolioService.get_user_portfolios(
+            user_id=current_user.id, include_inactive=include_inactive
+        )
+
+        return ResponseModel(
+            success=True,
+            message="Portfolios retrieved successfully",
+            data={
+                "portfolios": [
+                    await Portfolio_Pydantic.from_tortoise_orm(p) for p in portfolios
+                ],
+                "total_count": len(portfolios),
+            },
+        )
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.post("/", response_model=ResponseModel)
+async def create_portfolio(
+    portfolio_data: PortfolioCreate, current_user=Depends(get_current_user)
+):
+    try:
+        portfolio = await PortfolioService.create_portfolio(
+            user_id=current_user.id,
+            name=portfolio_data.name,
+            description=portfolio_data.description,
+        )
+        portfolio_pydantic_data = await Portfolio_Pydantic.from_tortoise_orm(portfolio)
+        return ResponseModel(
+            success=True,
+            message="Portfolio created successfully",
+            data=portfolio_pydantic_data,
+        )
+    except Exception as e:
+        if "unique constraint" in str(e).lower() or "UNIQUE constraint failed" in str(
+            e
+        ):
+            raise AppException(status_code=400, detail="Portfolio name already exists")
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.get("/{portfolio_id}", response_model=ResponseModel)
+async def get_portfolio_summary_route(  # Renamed to avoid conflict with service method
+    portfolio_id: int, current_user=Depends(get_current_user)
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id
+        )
+        if not portfolio:
+            raise AppException(status_code=404, detail="Portfolio not found")
+
+        summary = await PortfolioService.get_portfolio_summary(portfolio_id)
+
+        return ResponseModel(
+            success=True,
+            message="Portfolio summary retrieved successfully",
+            data=summary,
+        )
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.put("/{portfolio_id}", response_model=ResponseModel)
+async def update_portfolio(
+    portfolio_id: int,
+    portfolio_data: PortfolioUpdate,
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id
+        )
+        if not portfolio:
+            raise AppException(status_code=404, detail="Portfolio not found")
+
+        update_data = portfolio_data.dict(exclude_unset=True)
+        if update_data:
+            await portfolio.update_from_dict(update_data).save()
+
+        portfolio_pydantic_data = await Portfolio_Pydantic.from_tortoise_orm(portfolio)
+        return ResponseModel(
+            success=True,
+            message="Portfolio updated successfully",
+            data=portfolio_pydantic_data,
+        )
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.delete("/{portfolio_id}", response_model=ResponseModel)
+async def delete_portfolio(portfolio_id: int, current_user=Depends(get_current_user)):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id
+        )
+        if not portfolio:
+            raise AppException(status_code=404, detail="Portfolio not found")
+
+        portfolio.is_active = False
+        await portfolio.save()
+
+        return ResponseModel(
+            success=True,
+            message="Portfolio deleted successfully (set to inactive)",
+            data=None,
+        )
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+# Stock Holdings Routes
+
+
+@router.post(
+    "/{portfolio_id}/stocks",
+    response_model=ResponseModel,
+    summary="Buy/Add Stock to Portfolio",
+)
+async def add_stock_to_portfolio_route(  # Renamed
+    portfolio_id: int,
+    stock_data: StockHoldingCreate,
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id, is_active=True
+        )
+        if not portfolio:
+            raise AppException(
+                status_code=404, detail="Active portfolio not found or access denied"
+            )
+
+        holding = await PortfolioService.add_stock_to_portfolio(
+            portfolio_id=portfolio_id,
+            stock_id=stock_data.stock_id,
+            quantity_to_add=stock_data.quantity,
+            purchase_price_of_lot=stock_data.purchase_price,
+            purchase_date=stock_data.purchase_date,
+            notes=stock_data.notes,
+        )
+        # Convert full holding with related stock to response model if needed, or use Pydantic ORM model
+        # For simplicity, using the Pydantic model from ORM.
+        # The PortfolioStock_Pydantic might not include stock_symbol, stock_name if not configured.
+        # Re-fetch for full response if needed or ensure PortfolioStock_Pydantic has nested details.
+        # For now, assume PortfolioStock_Pydantic is sufficient.
+        holding_pydantic_data = await PortfolioStock_Pydantic.from_tortoise_orm(holding)
+        return ResponseModel(
+            success=True,
+            message="Stock bought and added/updated in portfolio successfully",
+            data=holding_pydantic_data,  # This will be the ORM model, not StockHoldingResponse
+        )
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.post(
+    "/{portfolio_id}/stocks/{stock_id}/sell",
+    response_model=ResponseModel,
+    summary="Sell Stock from Portfolio",
+)
+async def sell_stock_from_portfolio(
+    portfolio_id: int,
+    stock_id: int,  # stock_id identifies the asset
+    sell_data: StockSellSchema,
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id, is_active=True
+        )
+        if not portfolio:
+            raise AppException(
+                status_code=404, detail="Active portfolio not found or access denied"
+            )
+
+        transaction = await PortfolioService.sell_stock_holding(
+            portfolio_id=portfolio_id,
+            stock_id=stock_id,  # Pass stock_id from path
+            quantity_to_sell=sell_data.quantity,
+            sell_price=sell_data.sell_price,
+            sell_date=sell_data.sell_date,
+            notes=sell_data.notes,
+        )
+        transaction_pydantic_data = (
+            await PortfolioTransaction_Pydantic.from_tortoise_orm(transaction)
+        )
+        return ResponseModel(
+            success=True,
+            message="Stock sold successfully",
+            data=transaction_pydantic_data,
+        )
+    except DoesNotExist as e:
+        raise AppException(status_code=404, detail=str(e))
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.put("/{portfolio_id}/stocks/{stock_id}", response_model=ResponseModel)
+async def update_stock_holding(
+    portfolio_id: int,
+    stock_id: int,  # Changed from holding_id to stock_id
+    stock_data: StockHoldingUpdate,  # Be cautious with fields updated here for aggregated holdings
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id
+        )
+        if not portfolio:
+            raise AppException(status_code=404, detail="Portfolio not found")
+
+        # Fetch aggregated holding by stock_id and portfolio_id
+        holding = await PortfolioStock.get_or_none(
+            stock_id=stock_id, portfolio_id=portfolio_id
+        )
+        if not holding:
+            raise AppException(
+                status_code=404,
+                detail="Stock holding for this stock not found in portfolio.",
+            )
+
+        update_data = stock_data.dict(exclude_unset=True)
+        # Warning: Updating quantity/purchase_price/purchase_date directly on aggregated holding
+        # might lead to inconsistencies if not handled with proper recalculation logic.
+        # This endpoint should primarily be for 'notes' or very specific adjustments.
+        if (
+            "quantity" in update_data
+            or "purchase_price" in update_data
+            or "purchase_date" in update_data
+        ):
+            # Consider adding specific service methods for these adjustments if complex logic is needed.
+            pass  # Allowing direct update for now.
+
+        if update_data:
+            await holding.update_from_dict(update_data).save()
+
+        holding_pydantic_data = await PortfolioStock_Pydantic.from_tortoise_orm(holding)
+        return ResponseModel(
+            success=True,
+            message="Stock holding updated successfully",
+            data=holding_pydantic_data,
+        )
+    except DoesNotExist as e:  # Should be caught by the get_or_none checks
+        raise AppException(status_code=404, detail=str(e))
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.delete(
+    "/{portfolio_id}/stocks/{stock_id}",
+    response_model=ResponseModel,
+    summary="Delete Stock Holding",
+)
+async def remove_stock_from_portfolio(
+    portfolio_id: int,
+    stock_id: int,  # Changed from holding_id to stock_id
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id
+        )
+        if not portfolio:
+            raise AppException(status_code=404, detail="Portfolio not found")
+
+        success = await PortfolioService.remove_holding(
+            portfolio_id=portfolio_id,
+            asset_type_str="STOCK",
+            asset_id_value=stock_id,  # Use stock_id as asset_id_value
+        )
+        if not success:
+            raise AppException(
+                status_code=404,
+                detail="Stock holding not found or could not be deleted",
+            )
+
+        return ResponseModel(
+            success=True,
+            message="Stock holding removed from portfolio successfully",
+            data=None,
+        )
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+# UTT Holdings Routes
+
+
+@router.post(
+    "/{portfolio_id}/utts",
+    response_model=ResponseModel,
+    summary="Buy/Add UTT to Portfolio",
+)
+async def add_utt_to_portfolio_route(  # Renamed
+    portfolio_id: int,
+    utt_data: UTTHoldingCreate,
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id, is_active=True
+        )
+        if not portfolio:
+            raise AppException(
+                status_code=404, detail="Active portfolio not found or access denied"
+            )
+
+        holding = await PortfolioService.add_utt_to_portfolio(
+            portfolio_id=portfolio_id,
+            utt_fund_id=utt_data.utt_fund_id,
+            units_to_add=utt_data.units_held,
+            purchase_price_of_lot=utt_data.purchase_price,
+            purchase_date=utt_data.purchase_date,
+            notes=utt_data.notes,
+        )
+        holding_pydantic_data = await PortfolioUTT_Pydantic.from_tortoise_orm(holding)
+        return ResponseModel(
+            success=True,
+            message="UTT fund bought and added/updated in portfolio successfully",
+            data=holding_pydantic_data,
+        )
+    except DoesNotExist as e:
+        raise AppException(status_code=404, detail=str(e))
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.post(
+    "/{portfolio_id}/utts/{utt_fund_id}/sell",
+    response_model=ResponseModel,
+    summary="Sell UTT from Portfolio",
+)
+async def sell_utt_from_portfolio(
+    portfolio_id: int,
+    utt_fund_id: int,  # Changed from holding_id to utt_fund_id
+    sell_data: UTTSellSchema,
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id, is_active=True
+        )
+        if not portfolio:
+            raise AppException(
+                status_code=404, detail="Active portfolio not found or access denied"
+            )
+
+        transaction = await PortfolioService.sell_utt_holding(
+            portfolio_id=portfolio_id,
+            utt_fund_id=utt_fund_id,  # Use utt_fund_id from path
+            units_to_sell=sell_data.units_to_sell,  # Ensure schema field name is correct
+            sell_price=sell_data.sell_price,
+            sell_date=sell_data.sell_date,
+            notes=sell_data.notes,
+        )
+        transaction_pydantic_data = (
+            await PortfolioTransaction_Pydantic.from_tortoise_orm(transaction)
+        )
+        return ResponseModel(
+            success=True,
+            message="UTT units sold successfully",
+            data=transaction_pydantic_data,
+        )
+    except DoesNotExist as e:
+        raise AppException(status_code=404, detail=str(e))
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.put("/{portfolio_id}/utts/{utt_fund_id}", response_model=ResponseModel)
+async def update_utt_holding(
+    portfolio_id: int,
+    utt_fund_id: int,  # Changed from holding_id to utt_fund_id
+    utt_data: UTTHoldingUpdate,
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id
+        )
+        if not portfolio:
+            raise AppException(status_code=404, detail="Portfolio not found")
+
+        holding = await PortfolioUTT.get_or_none(
+            utt_fund_id=utt_fund_id, portfolio_id=portfolio_id
+        )
+        if not holding:
+            raise AppException(
+                status_code=404,
+                detail="UTT holding for this fund not found in portfolio.",
+            )
+
+        update_data = utt_data.dict(exclude_unset=True)
+        # Similar caution as with stock update for critical fields.
+        if (
+            "units_held" in update_data
+            or "purchase_price" in update_data
+            or "purchase_date" in update_data
+        ):
+            pass  # Allowing direct update
+
+        if update_data:
+            await holding.update_from_dict(update_data).save()
+
+        holding_pydantic_data = await PortfolioUTT_Pydantic.from_tortoise_orm(holding)
+        return ResponseModel(
+            success=True,
+            message="UTT holding updated successfully",
+            data=holding_pydantic_data,
+        )
+    except DoesNotExist as e:
+        raise AppException(status_code=404, detail=str(e))
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.delete(
+    "/{portfolio_id}/utts/{utt_fund_id}",
+    response_model=ResponseModel,
+    summary="Delete UTT Holding",
+)
+async def remove_utt_from_portfolio(
+    portfolio_id: int,
+    utt_fund_id: int,  # Changed from holding_id to utt_fund_id
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id
+        )
+        if not portfolio:
+            raise AppException(status_code=404, detail="Portfolio not found")
+
+        success = await PortfolioService.remove_holding(
+            portfolio_id=portfolio_id,
+            asset_type_str="UTT",
+            asset_id_value=utt_fund_id,  # Use utt_fund_id
+        )
+        if not success:
+            raise AppException(
+                status_code=404, detail="UTT holding not found or could not be deleted"
+            )
+
+        return ResponseModel(
+            success=True,
+            message="UTT fund holding removed from portfolio successfully",
+            data=None,
+        )
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+# Bond Holdings Routes
+
+
+@router.post(
+    "/{portfolio_id}/bonds",
+    response_model=ResponseModel,
+    summary="Buy/Add Bond to Portfolio",
+)
+async def add_bond_to_portfolio_route(  # Renamed
+    portfolio_id: int,
+    bond_data: BondHoldingCreate,  # Assumes bond_data.purchase_price is TOTAL cost
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id, is_active=True
+        )
+        if not portfolio:
+            raise AppException(
+                status_code=404, detail="Active portfolio not found or access denied"
+            )
+
+        _bond = await Bond.get_or_none(auction_number=bond_data.auction_number)
+        holding = await PortfolioService.add_bond_to_portfolio(
+            portfolio_id=portfolio_id,
+            bond_id=_bond.id,
+            face_value_to_add=bond_data.face_value_held,
+            total_purchase_price_of_lot=bond_data.purchase_price,  # Assumed total cost from schema
+            purchase_date=bond_data.purchase_date,
+            notes=bond_data.notes,
+        )
+        holding_pydantic_data = await PortfolioBond_Pydantic.from_tortoise_orm(holding)
+        return ResponseModel(
+            success=True,
+            message="Bond bought and added/updated in portfolio successfully",
+            data=holding_pydantic_data,
+        )
+    except DoesNotExist as e:
+        raise AppException(status_code=404, detail=str(e))
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.post(
+    "/{portfolio_id}/bonds/{bond_id}/sell",
+    response_model=ResponseModel,
+    summary="Sell Bond from Portfolio",
+)
+async def sell_bond_from_portfolio(
+    portfolio_id: int,
+    bond_id: int,  # Changed from holding_id to bond_id
+    sell_data: BondSellSchema,  # Assumes sell_data.sell_price is TOTAL proceeds
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id, is_active=True
+        )
+        if not portfolio:
+            raise AppException(
+                status_code=404, detail="Active portfolio not found or access denied"
+            )
+
+        transaction = await PortfolioService.sell_bond_holding(
+            portfolio_id=portfolio_id,
+            bond_id=bond_id,  # Use bond_id from path
+            face_value_to_sell=sell_data.face_value_to_sell,
+            sell_price_total=sell_data.sell_price,  # Assumed total proceeds from schema
+            sell_date=sell_data.sell_date,
+            notes=sell_data.notes,
+        )
+        transaction_pydantic_data = (
+            await PortfolioTransaction_Pydantic.from_tortoise_orm(transaction)
+        )
+        return ResponseModel(
+            success=True,
+            message="Bond portion sold successfully",
+            data=transaction_pydantic_data,
+        )
+    except DoesNotExist as e:
+        raise AppException(status_code=404, detail=str(e))
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.put("/{portfolio_id}/bonds/{bond_id}", response_model=ResponseModel)
+async def update_bond_holding(
+    portfolio_id: int,
+    bond_id: int,  # Changed from holding_id to bond_id
+    bond_data: BondHoldingUpdate,
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id
+        )
+        if not portfolio:
+            raise AppException(status_code=404, detail="Portfolio not found")
+
+        holding = await PortfolioBond.get_or_none(
+            bond_id=bond_id, portfolio_id=portfolio_id
+        )
+        if not holding:
+            raise AppException(
+                status_code=404,
+                detail="Bond holding for this bond not found in portfolio.",
+            )
+
+        update_data = bond_data.dict(exclude_unset=True)
+        # Caution: Updating face_value_held or purchase_price (total cost) directly
+        # should be done carefully. If face_value_held changes, purchase_price (total)
+        # should ideally be adjusted proportionally to maintain average cost per unit of FV,
+        # unless it's a specific correction.
+        if (
+            "face_value_held" in update_data
+            and "purchase_price" not in update_data
+            and holding.face_value_held > 0
+        ):
+            # If only face_value_held is changing, adjust purchase_price proportionally
+            # This is complex for a simple PUT, better handled by specific service method or by requiring both.
+            # For now, if only FV changes, the total cost is NOT proportionally adjusted here.
+            # User would need to provide new total purchase_price if FV changes and cost basis needs adjustment.
+            pass
+        elif "purchase_price" in update_data:  # Allows direct update of total cost
+            pass
+
+        if update_data:
+            await holding.update_from_dict(update_data).save()
+
+        holding_pydantic_data = await PortfolioBond_Pydantic.from_tortoise_orm(holding)
+        return ResponseModel(
+            success=True,
+            message="Bond holding updated successfully",
+            data=holding_pydantic_data,
+        )
+    except DoesNotExist as e:
+        raise AppException(status_code=404, detail=str(e))
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.delete(
+    "/{portfolio_id}/bonds/{bond_id}",
+    response_model=ResponseModel,
+    summary="Delete Bond Holding",
+)
+async def remove_bond_from_portfolio(
+    portfolio_id: int,
+    bond_id: int,  # Changed from holding_id to bond_id
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id
+        )
+        if not portfolio:
+            raise AppException(status_code=404, detail="Portfolio not found")
+
+        success = await PortfolioService.remove_holding(
+            portfolio_id=portfolio_id,
+            asset_type_str="BOND",
+            asset_id_value=bond_id,  # Use bond_id
+        )
+        if not success:
+            raise AppException(
+                status_code=404, detail="Bond holding not found or could not be deleted"
+            )
+
+        return ResponseModel(
+            success=True,
+            message="Bond holding removed from portfolio successfully",
+            data=None,
+        )
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+# Calendar and Transaction Routes (No changes related to holding_id vs asset_id here)
+
+
+@router.post("/{portfolio_id}/calendar", response_model=ResponseModel)
+async def add_calendar_event(
+    portfolio_id: int,
+    event_data: CalendarEventCreate,
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id
+        )
+        if not portfolio:
+            raise AppException(status_code=404, detail="Portfolio not found")
+
+        event = await PortfolioCalendar.create(
+            portfolio_id=portfolio_id, **event_data.dict()
+        )
+        event_pydantic_data = await PortfolioCalendar_Pydantic.from_tortoise_orm(event)
+        return ResponseModel(
+            success=True,
+            message="Calendar event added successfully",
+            data=event_pydantic_data,
+        )
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+@router.get("/{portfolio_id}/transactions", response_model=ResponseModel)
+async def get_portfolio_transactions(
+    portfolio_id: int,
+    limit: int = Query(50, ge=1, le=200),
+    offset: int = Query(0, ge=0),
+    current_user=Depends(get_current_user),
+):
+    try:
+        # 1. VALIDATION AND INITIAL QUERY (Same as before)
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id
+        )
+        if not portfolio:
+            raise AppException(status_code=404, detail="Portfolio not found")
+
+        transactions_query = (
+            PortfolioTransaction.filter(portfolio_id=portfolio_id)
+            .order_by("-transaction_date", "-created_at")
+            .offset(offset)
+            .limit(limit)
+        )
+        transactions_list = await transactions_query.all()
+
+        # --- ENRICHMENT LOGIC STARTS HERE ---
+
+        # 2. COLLECT UNIQUE ASSET IDs FROM THE TRANSACTION LIST
+        stock_ids = set()
+        utt_ids = set()
+        bond_ids = set()
+
+        for t in transactions_list:
+            if t.asset_type == "STOCK":
+                stock_ids.add(t.asset_id)
+            elif t.asset_type == "UTT":
+                utt_ids.add(t.asset_id)
+            elif t.asset_type == "BOND":
+                bond_ids.add(t.asset_id)
+
+        # 3. BULK FETCH ASSET DETAILS
+        stocks_map: Dict[int, Stock] = {
+            s.id: s for s in await Stock.filter(id__in=list(stock_ids))
+        }
+        utts_map: Dict[int, UTTFund] = {
+            u.id: u for u in await UTTFund.filter(id__in=list(utt_ids))
+        }
+        bonds_map: Dict[int, Bond] = {
+            b.id: b for b in await Bond.filter(id__in=list(bond_ids))
+        }
+
+        # 4. CONSTRUCT THE ENRICHED RESPONSE
+        enriched_transactions: List[TransactionDetailResponse] = []
+        for t in transactions_list:
+            asset_name = None
+            asset_symbol = None
+
+            if t.asset_type == "STOCK" and t.asset_id in stocks_map:
+                asset_name = stocks_map[t.asset_id].name
+                asset_symbol = stocks_map[t.asset_id].symbol
+            elif t.asset_type == "UTT" and t.asset_id in utts_map:
+                asset_name = utts_map[t.asset_id].name
+                asset_symbol = utts_map[t.asset_id].symbol
+            elif t.asset_type == "BOND" and t.asset_id in bonds_map:
+                bond = bonds_map[t.asset_id]
+                asset_name = f"{bond.maturity_years} Yr Treasury Bond"
+                asset_symbol = bond.isin
+
+            # Create the enriched Pydantic model
+            enriched_transaction = TransactionDetailResponse.model_validate(
+                {
+                    **t.__dict__,  # Unpack the transaction's own fields
+                    "asset_name": asset_name,
+                    "asset_symbol": asset_symbol,
+                }
+            )
+            enriched_transactions.append(enriched_transaction)
+
+        # --- ENRICHMENT LOGIC ENDS ---
+
+        total_count = await PortfolioTransaction.filter(
+            portfolio_id=portfolio_id
+        ).count()
+
+        return ResponseModel(
+            success=True,
+            message="Transactions retrieved successfully",
+            data={
+                # Use the new enriched list
+                "transactions": [et.model_dump() for et in enriched_transactions],
+                "total_count": total_count,
+                "limit": limit,
+                "offset": offset,
+            },
+        )
+    except Exception as e:
+        raise AppException(status_code=500, detail=str(e))
+
+
+# Performance and Analytics Routes
+
+
+@router.get(
+    "/{portfolio_id}/positions",
+    response_model=ResponseModel,
+    summary="Get All Current Portfolio Positions",
+)
+async def get_portfolio_positions(
+    portfolio_id: int, current_user=Depends(get_current_user)
+):
+    """
+    Calculates and retrieves all current positions in a portfolio.
+    It processes all buy/sell transactions to determine average cost,
+    fetches the latest market price, and calculates current value and profit/loss.
+    """
+    # 1. AUTHENTICATION & VALIDATION
+    portfolio = await Portfolio.get_or_none(id=portfolio_id, user_id=current_user.id)
+    if not portfolio:
+        raise AppException(status_code=404, detail="Portfolio not found")
+
+    # 2. FETCH AND AGGREGATE TRANSACTIONS
+    transactions = await PortfolioTransaction.filter(
+        portfolio_id=portfolio_id
+    ).order_by("transaction_date")
+
+    # This dictionary will hold the aggregated data for each asset
+    # Key: (asset_type, asset_id), Value: {buy_qty, buy_cost, sell_qty}
+    aggregated_data: Dict[tuple, Dict] = {}
+
+    for t in transactions:
+        asset_key = (t.asset_type, t.asset_id)
+        if asset_key not in aggregated_data:
+            aggregated_data[asset_key] = {
+                "buy_qty": Decimal("0.0"),
+                "buy_cost": Decimal("0.0"),
+                "sell_qty": Decimal("0.0"),
+            }
+
+        if t.transaction_type == "BUY":
+            aggregated_data[asset_key]["buy_qty"] += t.quantity
+            aggregated_data[asset_key]["buy_cost"] += t.total_amount
+        elif t.transaction_type == "SELL":
+            aggregated_data[asset_key]["sell_qty"] += t.quantity
+
+    # 3. PROCESS AGGREGATES AND FETCH LIVE DATA
+    position_responses: List[PositionResponse] = []
+
+    for asset_key, data in aggregated_data.items():
+        asset_type, asset_id = asset_key
+
+        current_quantity = data["buy_qty"] - data["sell_qty"]
+
+        # If the asset has been completely sold, skip it.
+        if current_quantity <= 0:
+            continue
+
+        # Calculate cost basis for the currently held units
+        avg_buy_price = (
+            data["buy_cost"] / data["buy_qty"]
+            if data["buy_qty"] > 0
+            else Decimal("0.0")
+        )
+        total_invested = current_quantity * avg_buy_price
+
+        # Fetch current price and asset details based on type
+        current_price = Decimal("0.0")
+        asset_name = "Unknown"
+        asset_symbol = "N/A"
+
+        if asset_type == "STOCK":
+            stock = await Stock.get_or_none(id=asset_id)
+            if stock:
+                asset_name = stock.name
+                asset_symbol = stock.symbol
+                price_data = (
+                    await StockPriceData.filter(stock_id=asset_id)
+                    .order_by("-date")
+                    .first()
+                )
+                if price_data:
+                    current_price = price_data.closing_price
+
+        elif asset_type == "UTT":
+            utt = await UTTFund.get_or_none(id=asset_id)
+            if utt:
+                asset_name = utt.name
+                asset_symbol = utt.symbol
+                price_data = (
+                    await UTTFundData.filter(fund_id=asset_id).order_by("-date").first()
+                )
+                if price_data:
+                    current_price = Decimal(str(price_data.nav_per_unit))
+
+        elif asset_type == "BOND":
+            bond = await Bond.get_or_none(id=asset_id)
+            if bond:
+                asset_name = f"{bond.maturity_years} Yr Treasury Bond"
+                asset_symbol = bond.isin
+                # Bond valuation is complex. We'll use a simplified assumption that the
+                # "price" is 100 for valuation purposes against its face value.
+                # Here, we'll represent price_per_100.
+                current_price = (
+                    Decimal(str(bond.price_per_100))
+                    if bond.price_per_100
+                    else Decimal("100.0")
+                )
+
+        # Calculate final metrics
+        current_value = current_quantity * current_price
+        profit_loss = current_value - total_invested
+        profit_loss_percent = (
+            (profit_loss / total_invested) * 100 if total_invested > 0 else 0.0
+        )
+
+        # Create the response object
+        position = PositionResponse(
+            asset_id=asset_id,
+            asset_type=asset_type.capitalize(),  # "STOCK" -> "Stock"
+            asset_name=asset_name,
+            asset_symbol=asset_symbol,
+            quantity=current_quantity,
+            avg_buy_price=round(avg_buy_price, 4),
+            total_invested=round(total_invested, 2),
+            current_price=round(current_price, 4),
+            current_value=round(current_value, 2),
+            profit_loss=round(profit_loss, 2),
+            profit_loss_percent=round(float(profit_loss_percent), 2),
+        )
+        position_responses.append(position)
+
+    # 4. RETURN THE FINAL RESPONSE
+    return ResponseModel(
+        success=True,
+        message="Positions retrieved successfully.",
+        data={"positions": position_responses},
+    )
+
+
+@router.post("/{portfolio_id}/snapshot", response_model=ResponseModel)
+async def create_portfolio_snapshot_route(  # Renamed
+    portfolio_id: int,
+    snapshot_date: Optional[date] = Query(
+        None, description="Date for the snapshot. Defaults to today if None."
+    ),
+    current_user=Depends(get_current_user),
+):
+    try:
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id
+        )
+        if not portfolio:
+            raise AppException(status_code=404, detail="Portfolio not found")
+
+        snapshot_orm = await PortfolioService.create_portfolio_snapshot(
+            portfolio_id=portfolio_id, snapshot_date_input=snapshot_date
+        )
+
+        snapshot_pydantic_data = await PortfolioSnapshotPydantic.from_tortoise_orm(
+            snapshot_orm
+        )
+
+        return ResponseModel(
+            success=True,
+            message="Portfolio snapshot created successfully",
+            data=snapshot_pydantic_data,
+        )
+    except NotImplementedError as e:  # Catch specific error from service
+        raise AppException(status_code=501, detail=str(e))
+    except DoesNotExist:
+        raise AppException(
+            status_code=404, detail="Portfolio not found when creating snapshot."
+        )
+    except AppException:
+        raise
+    except Exception as e:
+        raise AppException(
+            status_code=500, detail=f"Failed to create snapshot: {str(e)}"
+        )
+
+
+@router.get(
+    "/{portfolio_id}/performance",
+    response_model=ResponseModel,
+    summary="Get Portfolio Performance Timeseries",
+)
+async def get_portfolio_performance(
+    portfolio_id: int,
+    background_tasks: BackgroundTasks,
+    period: str = Query(
+        "1M",
+        enum=["1D", "1W", "1M", "YTD", "1Y", "Max"],
+        description="The time period for the performance data.",
+    ),
+    current_user=Depends(get_current_user),
+):
+    """
+    Retrieves time-series performance data for a portfolio.
+    If data is missing, it automatically queues a background task to generate
+    all historical data and informs the user to wait.
+    """
+    try:
+        # 1. AUTHENTICATION
+        portfolio = await Portfolio.get_or_none(
+            id=portfolio_id, user_id=current_user.id
+        )
+        if not portfolio:
+            raise AppException(status_code=404, detail="Portfolio not found")
+
+        # 2. CONSOLIDATED TASK CHECK: Check if ANY relevant task is already running.
+        active_task = await ImportTask.filter(
+            Q(details__contains={"portfolio_id": portfolio_id}),
+            Q(task_type__in=["portfolio_regeneration", "portfolio_snapshot_history"]),
+            status__in=["pending", "running"],
+        ).first()
+
+        if active_task:
+            return ResponseModel(
+                success=False,
+                message="Portfolio performance data is currently being prepared. Please check back in a few moments.",
+                data={"task_id": active_task.id, "status": active_task.status},
+            )
+
+        # 3. DEFINE TIME PERIOD & QUERY EXISTING DATA
+        end_date = date.today()
+        start_date = None
+        if period == "1D":
+            start_date = end_date - timedelta(days=1)
+        elif period == "1W":
+            start_date = end_date - timedelta(weeks=1)
+        elif period == "1M":
+            start_date = end_date - timedelta(days=30)
+        elif period == "YTD":
+            start_date = date(end_date.year, 1, 1)
+        elif period == "1Y":
+            start_date = end_date - timedelta(days=365)
+        if period == "Max":
+            start_date = end_date - timedelta(days=365 * 10)  # A 10-year fallback
+
+        query = PortfolioSnapshot.filter(portfolio_id=portfolio_id)
+        if start_date:
+            start_datetime = datetime.combine(start_date, datetime.min.time())
+            query = query.filter(snapshot_date__gte=start_datetime)
+
+        snapshots = await query.order_by("snapshot_date").values(
+            "snapshot_date", "total_value"
+        )
+
+        #### delete snapshots ####
+
+        # 4. DECISION POINT: Serve data OR trigger generation.
+        # If we found no snapshots for the requested period, it's time to generate.
+        if not snapshots:
+            # Since we already checked for active tasks, we know it's safe to start a new one.
+            task = await ImportTask.create(
+                task_type="portfolio_snapshot_history",
+                status="pending",
+                details={
+                    "portfolio_id": portfolio_id,
+                    "reason": "First-time data request.",
+                },
+            )
+            # We call the task without a start_date, so it will find the earliest transaction.
+            background_tasks.add_task(
+                PortfolioService.regenerate_snapshots_task, task.id, portfolio_id
+            )
+
+            return ResponseModel(
+                success=False,
+                message="We're preparing your performance history for the first time. This may take a moment.",
+                data={"task_id": task.id, "status": "pending"},
+            )
+
+        # 5. SUCCESS PATH: This code is only reached if snapshots WERE found.
+        if len(snapshots) < 2:
+            current_value = snapshots[0]["total_value"]
+            return ResponseModel(
+                success=True,
+                message="Not enough historical data to calculate performance change.",
+                data={
+                    "current_value": str(current_value),
+                    "change_value": "0.00",
+                    "change_percentage": 0.0,
+                    "timeseries": [
+                        {
+                            "date": s["snapshot_date"].isoformat(),
+                            "value": str(s["total_value"]),
+                        }
+                        for s in snapshots
+                    ],
+                },
+            )
+
+        first_value = snapshots[0]["total_value"]
+        last_value = snapshots[-1]["total_value"]
+        change_value = last_value - first_value
+        change_percentage = (
+            (change_value / first_value) * 100 if first_value > 0 else Decimal("0.0")
+        )
+
+        return ResponseModel(
+            success=True,
+            message=f"Performance data for period '{period}' retrieved successfully.",
+            data={
+                "current_value": str(last_value),
+                "change_value": str(change_value),
+                "change_percentage": round(float(change_percentage), 2),
+                "timeseries": [
+                    {
+                        "date": s["snapshot_date"].isoformat(),
+                        "value": str(s["total_value"]),
+                    }
+                    for s in snapshots
+                ],
+            },
+        )
+
+    except Exception as e:
+        raise AppException(status_code=500, detail=f"An unexpected error occurred: {e}")
+
+
+@router.get(
+    "/{portfolio_id}/calendar",
+    response_model=ResponseModel,
+    summary="Get Upcoming Portfolio Calendar Events",
+)
+async def get_portfolio_calendar_events(
+    portfolio_id: int,
+    start_date: Optional[date] = Query(
+        None, description="Start of date range. Defaults to today."
+    ),
+    end_date: Optional[date] = Query(
+        None, description="End of date range. Defaults to 90 days from now."
+    ),
+    current_user=Depends(get_current_user),
+):
+    """
+    Generates a dynamic calendar of expected income events (dividends and coupons)
+    for a user's portfolio within a given date range.
+    """
+    # 1. SETUP & AUTHENTICATION
+    if start_date is None:
+        start_date = date.today()
+    if end_date is None:
+        end_date = start_date + timedelta(days=90)
+
+    portfolio = await Portfolio.get_or_none(id=portfolio_id, user_id=current_user.id)
+    if not portfolio:
+        raise AppException(status_code=404, detail="Portfolio not found")
+
+    calendar_events: List[CalendarEventResponse] = []
+    print("Hello there buddy!!")
+    # 2. PROCESS STOCK DIVIDENDS
+    # Get all stocks currently held in the portfolio
+    portfolio_stocks = await PortfolioStock.filter(
+        portfolio_id=portfolio_id
+    ).select_related("stock")
+    print("Hello there buddy!!")
+    if portfolio_stocks:
+        stock_ids = [ps.stock.id for ps in portfolio_stocks]
+
+        # Create a map for quick lookup of quantity held for each stock
+        stock_quantity_map = {ps.stock.id: ps.quantity for ps in portfolio_stocks}
+
+        # Find all declared dividends for those stocks within the date range
+        dividends = await Dividend.filter(
+            stock_id__in=stock_ids,
+            payment_date__gte=start_date,
+            payment_date__lte=end_date,
+        ).select_related("stock")
+        print(dividends)
+        for div in dividends:
+            quantity_held = stock_quantity_map.get(div.stock.id, 0)
+            if quantity_held > 0:
+                event = CalendarEventResponse(
+                    event_date=div.payment_date,
+                    event_type="Dividend Payment",
+                    asset_symbol=div.stock.symbol,
+                    asset_name=div.stock.name,
+                    estimated_amount=div.dividend_amount * quantity_held,
+                    notes=f"Ex-dividend date: {div.ex_dividend_date.isoformat()}",
+                )
+                calendar_events.append(event)
+
+    # 3. PROCESS BOND COUPONS
+    # Get all bonds currently held in the portfolio
+    portfolio_bonds = await PortfolioBond.filter(
+        portfolio_id=portfolio_id
+    ).select_related("bond")
+    if portfolio_bonds:
+        for pb in portfolio_bonds:
+            # Use our helper function to calculate coupon dates in the range
+            coupon_dates = _calculate_bond_coupon_dates(pb.bond, start_date, end_date)
+
+            for coupon_date in coupon_dates:
+                # Coupon amount is based on face value and semi-annual rate
+                estimated_amount = (
+                    pb.face_value_held
+                    * (Decimal(str(pb.bond.coupon_rate)) / Decimal("100"))
+                ) / Decimal("2")
+
+                event = CalendarEventResponse(
+                    event_date=coupon_date,
+                    event_type="Bond Coupon",
+                    asset_symbol=pb.bond.isin,
+                    asset_name=f"{pb.bond.maturity_years} Yr T-Bond",
+                    estimated_amount=estimated_amount,
+                    notes=f"Matures on {pb.bond.maturity_date.isoformat()}",
+                )
+                calendar_events.append(event)
+
+    # 4. SORT AND RETURN
+    # Sort all collected events by date
+    sorted_events = sorted(calendar_events, key=lambda x: x.event_date)
+
+    return ResponseModel(
+        success=True,
+        message="Portfolio calendar events retrieved successfully.",
+        data={"events": sorted_events, "total_count": len(sorted_events)},
+    )

App/routers/portfolio/schemas.py

@@ -0,0 +1,419 @@
+# schemas.py
+from pydantic import BaseModel, Field, ConfigDict  # Use ConfigDict for Pydantic V2
+from typing import Optional, List
+from datetime import date, datetime
+from decimal import Decimal
+
+
+# --- Portfolio Schemas ---
+class PortfolioBase(BaseModel):
+    id: int
+    name: str
+    description: Optional[str] = None
+    is_active: bool
+    created_at: datetime
+    updated_at: datetime
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class PortfolioCreate(BaseModel):
+    name: str = Field(
+        ..., min_length=1, max_length=100, description="Name of the portfolio"
+    )
+    description: Optional[str] = Field(
+        None, description="Optional description for the portfolio"
+    )
+
+
+class PortfolioUpdate(BaseModel):
+    name: Optional[str] = Field(
+        None, min_length=1, max_length=100, description="New name for the portfolio"
+    )
+    description: Optional[str] = Field(
+        None, description="New description for the portfolio"
+    )
+    is_active: Optional[bool] = Field(
+        None, description="Set portfolio active or inactive status"
+    )
+
+
+class PortfolioListResponse(BaseModel):
+    portfolios: List[PortfolioBase]
+    total_count: int
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+# --- Stock Holding Schemas ---
+class StockHoldingBase(BaseModel):
+    stock_id: int = Field(..., description="Internal ID of the stock master record")
+    quantity: Decimal = Field(..., gt=0, description="Number of shares held")
+    purchase_price: Decimal = Field(
+        ...,
+        gt=0,
+        description="Average price per share at purchase for the aggregated holding",
+    )
+    purchase_date: date = Field(
+        ..., description="Representative date of stock purchase (e.g., latest buy)"
+    )
+    notes: Optional[str] = Field(None, description="Additional notes for this holding")
+
+
+class StockHoldingCreate(StockHoldingBase):
+    # Used when adding a new lot of stocks. purchase_price is unit price for this lot.
+    pass
+
+
+class StockHoldingUpdate(BaseModel):
+    # For updating notes or other specific fields on an aggregated holding.
+    # Avoid direct updates to quantity/purchase_price here unless specific logic handles recalculation of average price.
+    quantity: Optional[Decimal] = Field(
+        None, gt=0, description="Updated total number of shares (use with caution)"
+    )
+    purchase_price: Optional[Decimal] = Field(
+        None,
+        gt=0,
+        description="Updated average purchase price per share (use with caution)",
+    )
+    purchase_date: Optional[date] = Field(
+        None, description="Updated representative purchase date"
+    )
+    notes: Optional[str] = Field(None, description="Updated notes")
+
+
+class StockHoldingResponse(StockHoldingBase):
+    id: int = Field(
+        ..., description="Unique ID of the PortfolioStock (aggregated holding) record"
+    )
+    stock_symbol: str = Field(..., description="Ticker symbol of the stock")
+    stock_name: str = Field(..., description="Name of the stock company")
+    current_price: Optional[Decimal] = Field(
+        None, description="Current market price per share"
+    )
+    market_value: Optional[Decimal] = Field(
+        None, description="Total current market value of the holding"
+    )
+    gain_loss: Optional[Decimal] = Field(None, description="Absolute gain or loss")
+    gain_loss_percentage: Optional[Decimal] = Field(
+        None, description="Percentage gain or loss"
+    )
+    created_at: datetime
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class StockSellSchema(BaseModel):
+    quantity: Decimal = Field(..., gt=0, description="Number of shares to sell")
+    sell_price: Decimal = Field(
+        ..., gt=0, description="Price per share at which stock was sold"
+    )
+    sell_date: date = Field(..., description="Date of the sale")
+    notes: Optional[str] = Field(
+        None, description="Additional notes for the sell transaction"
+    )
+
+
+# --- UTT (Unit Trust / Mutual Fund) Holding Schemas ---
+class UTTHoldingBase(BaseModel):
+    utt_fund_id: int = Field(
+        ..., description="Internal ID of the UTT fund master record"
+    )
+    units_held: Decimal = Field(..., gt=0, description="Number of units held")
+    purchase_price: Decimal = Field(
+        ...,
+        gt=0,
+        description="Average price per unit at purchase (NAV) for the aggregated holding",
+    )
+    purchase_date: date = Field(
+        ..., description="Representative date of UTT purchase (e.g., latest buy)"
+    )
+    notes: Optional[str] = Field(None, description="Additional notes for this holding")
+
+
+class UTTHoldingCreate(UTTHoldingBase):
+    # Used when adding a new lot of UTTs. purchase_price is unit price for this lot.
+    pass
+
+
+class UTTHoldingUpdate(BaseModel):
+    units_held: Optional[Decimal] = Field(
+        None, gt=0, description="Updated number of units held (use with caution)"
+    )
+    purchase_price: Optional[Decimal] = Field(
+        None,
+        gt=0,
+        description="Updated average purchase price per unit (use with caution)",
+    )
+    purchase_date: Optional[date] = Field(
+        None, description="Updated representative purchase date"
+    )
+    notes: Optional[str] = Field(None, description="Updated notes")
+
+
+class UTTHoldingResponse(UTTHoldingBase):
+    id: int = Field(
+        ..., description="Unique ID of the PortfolioUTT (aggregated holding) record"
+    )
+    fund_symbol: str = Field(..., description="Symbol of the UTT fund")
+    fund_name: str = Field(..., description="Name of the UTT fund")
+    current_nav: Optional[Decimal] = Field(
+        None, description="Current Net Asset Value (NAV) per unit"
+    )
+    market_value: Optional[Decimal] = Field(
+        None, description="Total current market value of the holding"
+    )
+    gain_loss: Optional[Decimal] = Field(None, description="Absolute gain or loss")
+    gain_loss_percentage: Optional[Decimal] = Field(
+        None, description="Percentage gain or loss"
+    )
+    created_at: datetime
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class UTTSellSchema(BaseModel):
+    units_to_sell: Decimal = Field(
+        ..., gt=0, description="Number of UTT units to sell"
+    )  # Changed from 'units'
+    sell_price: Decimal = Field(
+        ..., gt=0, description="Price per unit at which UTT was sold (NAV)"
+    )
+    sell_date: date = Field(..., description="Date of the sale")
+    notes: Optional[str] = Field(
+        None, description="Additional notes for the sell transaction"
+    )
+
+
+# --- Bond Holding Schemas ---
+class BondHoldingBase(BaseModel):
+    # bond_id: int = Field(..., description="Internal ID of the bond master record")
+    face_value_held: Decimal = Field(
+        ..., gt=0, description="Total face value of the bond held"
+    )
+    auction_number: Optional[int] = Field(
+        None, description="Auction number if applicable (e.g., for government bonds)"
+    )
+    auction_date: Optional[date] = Field(
+        None, description="Auction date if applicable (e.g., for government bonds)"
+    )
+    purchase_price: Decimal = Field(
+        ...,
+        gt=0,
+        description="TOTAL purchase price paid for the entire face_value_held (aggregated holding).",
+    )
+    purchase_date: date = Field(
+        ..., description="Representative date of bond purchase (e.g., latest buy)"
+    )
+    notes: Optional[str] = Field(None, description="Additional notes for this holding")
+
+
+class BondHoldingCreate(BondHoldingBase):
+    # Used when adding a new lot of bonds. purchase_price is TOTAL cost for this specific lot of face_value_held.
+    pass
+
+
+class BondHoldingUpdate(BaseModel):
+    face_value_held: Optional[Decimal] = Field(
+        None, gt=0, description="Updated total face value held"
+    )
+    purchase_price: Optional[Decimal] = Field(
+        None,
+        gt=0,
+        description="Updated TOTAL purchase price for the new face_value_held (use with caution)",
+    )
+    purchase_date: Optional[date] = Field(
+        None, description="Updated representative purchase date"
+    )
+    notes: Optional[str] = Field(None, description="Updated notes")
+
+
+class BondHoldingResponse(BondHoldingBase):
+    id: int = Field(
+        ..., description="Unique ID of the PortfolioBond (aggregated holding) record"
+    )
+    instrument_type: str = Field(..., description="Type of bond instrument")
+    auction_number: Optional[int] = Field(
+        None, description="Auction number if applicable"
+    )
+    maturity_date: date = Field(..., description="Maturity date of the bond")
+    current_price: Optional[Decimal] = Field(
+        None,
+        description="Current market price (e.g., percentage of face value like 99.5)",
+    )
+    market_value: Optional[Decimal] = Field(
+        None, description="Total current market value of the holding"
+    )
+    accrued_interest: Optional[Decimal] = Field(
+        None, description="Accrued interest on the bond"
+    )
+    yield_to_maturity: Optional[Decimal] = Field(
+        None, description="Yield to maturity of the bond"
+    )
+    gain_loss: Optional[Decimal] = Field(
+        None, description="Absolute gain or loss on principal"
+    )
+    created_at: datetime
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class BondSellSchema(BaseModel):
+    face_value_to_sell: Decimal = Field(
+        ..., gt=0, description="Face value of the bond portion being sold"
+    )  # Changed from 'face_value_sold'
+    sell_price: Decimal = Field(
+        ..., gt=0, description="TOTAL selling proceeds for the face_value_to_sell."
+    )
+    sell_date: date = Field(..., description="Date of the sale")
+    notes: Optional[str] = Field(
+        None, description="Additional notes for the sell transaction"
+    )
+
+
+# --- Calendar Event Schemas ---
+class CalendarEventBase(BaseModel):
+    event_date: date
+    event_type: str = Field(..., max_length=50)
+    title: str = Field(..., max_length=200)
+    description: Optional[str] = None
+    asset_type: Optional[str] = Field(None, max_length=10)
+    asset_id: Optional[int] = None
+    estimated_amount: Optional[Decimal] = None
+
+
+class CalendarEventCreate(CalendarEventBase):
+    pass
+
+
+class CalendarEventResponse(CalendarEventBase):
+    id: int
+    is_completed: bool = Field(False)
+    created_at: datetime
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+# --- Transaction Schemas ---
+class TransactionBase(BaseModel):
+    transaction_type: str = Field(..., max_length=20)
+    asset_type: str = Field(..., max_length=10)
+    asset_id: Optional[int] = None
+    asset_name: Optional[str] = Field(None, max_length=100)
+    quantity: Optional[Decimal] = None
+    price: Optional[Decimal] = Field(None, ge=0)
+    transaction_date: date
+    notes: Optional[str] = None
+
+
+class TransactionCreate(TransactionBase):
+    total_amount: Decimal  # Service layer calculates and provides this.
+
+
+class TransactionResponse(TransactionBase):
+    id: int
+    total_amount: Decimal
+    created_at: datetime
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+# --- Portfolio Analytics & Summary Schemas ---
+class AssetAllocation(BaseModel):
+    stocks_percentage: Decimal = Field(Decimal("0.0"), ge=0, le=100)
+    bonds_percentage: Decimal = Field(Decimal("0.0"), ge=0, le=100)
+    utts_percentage: Decimal = Field(Decimal("0.0"), ge=0, le=100)
+    cash_percentage: Decimal = Field(Decimal("0.0"), ge=0, le=100)
+    total_value: Decimal
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class CalendarEventResponse(BaseModel):
+    event_date: date
+    event_type: str  # e.g., "Dividend Payment", "Bond Coupon"
+    asset_symbol: str
+    asset_name: str
+    estimated_amount: Decimal
+    notes: Optional[str] = None
+
+    model_config = ConfigDict(
+        from_attributes=True,
+    )
+
+
+class TransactionDetailResponse(BaseModel):
+    id: int
+    transaction_type: str
+    asset_type: str
+    asset_id: int
+    quantity: Decimal
+    price: Decimal
+    total_amount: Decimal
+    transaction_date: date
+    notes: Optional[str] = None
+    created_at: datetime
+
+    # New fields to be added
+    asset_name: Optional[str] = None
+    asset_symbol: Optional[str] = None
+
+    model_config = ConfigDict(
+        from_attributes=True,
+    )
+
+
+class PortfolioSummary(BaseModel):
+    portfolio: PortfolioBase
+    total_market_value: Decimal
+    total_cost_basis: Decimal
+    overall_unrealized_gain_loss: Decimal
+    overall_unrealized_gain_loss_percentage: Decimal
+    stock_holdings: List[StockHoldingResponse] = Field(default_factory=list)
+    utt_holdings: List[UTTHoldingResponse] = Field(default_factory=list)
+    bond_holdings: List[BondHoldingResponse] = Field(default_factory=list)
+    asset_allocation: AssetAllocation
+    recent_transactions: List[TransactionResponse] = Field(default_factory=list)
+    upcoming_events: List[CalendarEventResponse] = Field(default_factory=list)
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class AssetPerformanceDetail(BaseModel):
+    asset_id: Optional[int] = None
+    name: str
+    return_value: Decimal
+    asset_type: Optional[str] = None
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class PortfolioPerformance(BaseModel):
+    portfolio_id: int
+    period: str
+    start_value: Decimal
+    end_value: Decimal
+    absolute_return: Decimal
+    percentage_return: Decimal
+    best_performer: Optional[AssetPerformanceDetail] = None
+    worst_performer: Optional[AssetPerformanceDetail] = None
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class PositionResponse(BaseModel):
+    asset_id: int
+    asset_type: str
+    asset_name: str
+    asset_symbol: str
+    quantity: Decimal
+    avg_buy_price: Decimal
+    total_invested: Decimal
+    current_price: Decimal
+    current_value: Decimal
+    profit_loss: Decimal
+    profit_loss_percent: float
+
+    model_config = ConfigDict(
+        from_attributes=True,
+    )

App/routers/portfolio/service.py

@@ -0,0 +1,996 @@
+# service.py
+from typing import List, Optional, Dict, Any
+from decimal import Decimal
+from datetime import date, datetime, timezone  # Added timezone
+from tortoise.exceptions import DoesNotExist
+from tortoise.transactions import in_transaction
+
+from App.schemas import AppException  # Assuming AppException is in App.schemas
+
+from .models import (
+    Portfolio,
+    PortfolioStock,
+    PortfolioUTT,
+    PortfolioBond,
+    PortfolioTransaction,
+    PortfolioCalendar,
+    PortfolioSnapshot,
+)
+
+# Assuming models for stocks, utts, bonds are in these paths
+from ..stocks.models import Stock, StockPriceData
+from ..utt.models import UTTFund, UTTFundData
+from ..bonds.models import (
+    Bond,
+)  # Assuming Bond model might have price_per_100 or similar
+
+# Import Pydantic schemas
+from .schemas import (
+    PortfolioSummary,
+    StockHoldingResponse,
+    UTTHoldingResponse,
+    BondHoldingResponse,
+    AssetAllocation,
+    PortfolioBase,
+    TransactionResponse,
+    CalendarEventResponse,  # Added PortfolioBase and other response schemas
+)
+
+from App.routers.tasks.models import ImportTask
+from datetime import date, timedelta
+from tortoise.expressions import Q
+from typing import List, Generator
+
+
+def _calculate_bond_coupon_dates(
+    bond: Bond, start_date: date, end_date: date
+) -> Generator[date, None, None]:
+    """
+    Calculates the semi-annual coupon payment dates for a bond within a given date range.
+
+    This makes a common assumption that coupon payments occur semi-annually,
+    with one payment on the maturity month/day and the other 6 months apart.
+    """
+    if bond.maturity_date and bond.coupon_rate > 0:
+        # First coupon payment month and day
+        month1 = bond.maturity_date.month
+        day1 = bond.maturity_date.day
+
+        # Second coupon payment is 6 months from the first
+        month2 = (
+            month1 + 5
+        ) % 12 + 1  # +5 then %12 handles the 6-month offset correctly
+
+        # Iterate through years from the bond's issue to maturity
+        for year in range(bond.effective_date.year, bond.maturity_date.year + 1):
+            try:
+                # Construct the two potential coupon dates for the year
+                coupon_date1 = date(year, month1, day1)
+                coupon_date2 = date(year, month2, day1)  # Day is assumed the same
+
+                # Yield the date if it falls within the user's requested filter range
+                if start_date <= coupon_date1 <= end_date:
+                    yield coupon_date1
+                if start_date <= coupon_date2 <= end_date:
+                    yield coupon_date2
+            except ValueError:
+                # Handles cases like Feb 29 on a non-leap year, just skip that invalid date.
+                continue
+
+
+class PortfolioService:
+
+    @staticmethod
+    async def get_user_portfolios(
+        user_id: int, include_inactive: bool = False
+    ) -> List[Portfolio]:
+        """Get all portfolios for a user"""
+        query = Portfolio.filter(user_id=user_id)
+        if not include_inactive:
+            query = query.filter(is_active=True)
+        return await query.order_by("-created_at").all()
+
+    @staticmethod
+    async def create_portfolio(
+        user_id: int, name: str, description: Optional[str] = None
+    ) -> Portfolio:
+        """Create a new portfolio for user"""
+        return await Portfolio.create(
+            user_id=user_id, name=name, description=description
+        )
+
+    @staticmethod
+    async def get_portfolio_summary(portfolio_id: int) -> PortfolioSummary:
+        """Get comprehensive portfolio summary with all holdings and calculations"""
+        portfolio_orm = await Portfolio.get_or_none(id=portfolio_id)
+        if not portfolio_orm:
+            raise DoesNotExist("Portfolio not found")
+
+        # Get all holdings with calculated values
+        stock_holdings_resp = await PortfolioService._get_stock_holdings_with_values(
+            portfolio_id
+        )
+        utt_holdings_resp = await PortfolioService._get_utt_holdings_with_values(
+            portfolio_id
+        )
+        bond_holdings_resp = await PortfolioService._get_bond_holdings_with_values(
+            portfolio_id
+        )
+
+        # Calculate total market values
+        total_stock_value = sum(
+            h.market_value or Decimal("0") for h in stock_holdings_resp
+        )
+        total_utt_value = sum(
+            Decimal(h.market_value) or Decimal("0") for h in utt_holdings_resp
+        )
+        total_bond_value = sum(
+            Decimal(h.market_value) or Decimal("0") for h in bond_holdings_resp
+        )
+        total_market_value = total_stock_value + total_utt_value + total_bond_value
+
+        # Calculate total cost basis
+        # For stocks/UTTs, purchase_price is average unit price on the aggregated holding.
+        total_stock_cost = sum(
+            h.purchase_price * h.quantity for h in stock_holdings_resp
+        )
+        total_utt_cost = sum(h.purchase_price * h.units_held for h in utt_holdings_resp)
+        # For bonds, BondHoldingResponse.purchase_price is the *total* purchase cost for that aggregated holding.
+        total_bond_cost = sum(h.purchase_price for h in bond_holdings_resp)
+        total_cost_basis = total_stock_cost + total_utt_cost + total_bond_cost
+
+        # Calculate overall gains/losses
+        overall_unrealized_gain_loss = total_market_value - total_cost_basis
+        overall_unrealized_gain_loss_percentage = (
+            (overall_unrealized_gain_loss / total_cost_basis * Decimal("100"))
+            if total_cost_basis > 0
+            else Decimal("0")
+        )
+
+        # Get recent transactions
+        recent_transactions_orm = (
+            await PortfolioTransaction.filter(portfolio_id=portfolio_id)
+            .order_by("-transaction_date", "-created_at")
+            .limit(10)
+            .all()
+        )
+        recent_transactions_resp = [
+            TransactionResponse.from_orm(t) for t in recent_transactions_orm
+        ]
+
+        # Get upcoming events
+        upcoming_events_orm = (
+            await PortfolioCalendar.filter(
+                portfolio_id=portfolio_id,
+                event_date__gte=date.today(),
+                is_completed=False,
+            )
+            .order_by("event_date")
+            .limit(10)
+            .all()
+        )
+        upcoming_events_resp = [
+            CalendarEventResponse.from_orm(e) for e in upcoming_events_orm
+        ]
+
+        # Asset allocation
+        asset_alloc = AssetAllocation(
+            stocks_percentage=(
+                (total_stock_value / total_market_value * Decimal("100"))
+                if total_market_value > 0
+                else Decimal("0")
+            ),
+            bonds_percentage=(
+                (total_bond_value / total_market_value * Decimal("100"))
+                if total_market_value > 0
+                else Decimal("0")
+            ),
+            utts_percentage=(
+                (total_utt_value / total_market_value * Decimal("100"))
+                if total_market_value > 0
+                else Decimal("0")
+            ),
+            cash_percentage=Decimal(
+                "0"
+            ),  # Assuming cash is not directly tracked here yet
+            total_value=total_market_value,
+        )
+
+        portfolio_base = PortfolioBase.from_orm(portfolio_orm)
+
+        return PortfolioSummary(
+            portfolio=portfolio_base,
+            total_market_value=total_market_value,
+            total_cost_basis=total_cost_basis,
+            overall_unrealized_gain_loss=overall_unrealized_gain_loss,
+            overall_unrealized_gain_loss_percentage=overall_unrealized_gain_loss_percentage,
+            stock_holdings=stock_holdings_resp,
+            utt_holdings=utt_holdings_resp,
+            bond_holdings=bond_holdings_resp,
+            asset_allocation=asset_alloc,
+            recent_transactions=recent_transactions_resp,
+            upcoming_events=upcoming_events_resp,
+        )
+
+    @staticmethod
+    async def _get_stock_holdings_with_values(
+        portfolio_id: int,
+    ) -> List[StockHoldingResponse]:
+        holdings_orm = (
+            await PortfolioStock.filter(portfolio_id=portfolio_id)
+            .prefetch_related("stock")
+            .all()
+        )
+        results = []
+        for holding in holdings_orm:  # holding is now an aggregated record
+            latest_price_data = (
+                await StockPriceData.filter(stock_id=holding.stock_id)
+                .order_by("-date")
+                .first()
+            )
+            current_price = (
+                latest_price_data.closing_price if latest_price_data else None
+            )
+
+            market_value = (
+                (current_price * holding.quantity)
+                if current_price is not None
+                else None
+            )
+            # holding.purchase_price is average unit price
+            cost_basis = holding.purchase_price * holding.quantity
+            gain_loss = (
+                (market_value - cost_basis) if market_value is not None else None
+            )
+            gain_loss_percentage = (
+                (gain_loss / cost_basis * Decimal("100"))
+                if gain_loss is not None and cost_basis > 0
+                else None
+            )
+
+            results.append(
+                StockHoldingResponse(
+                    id=holding.id,  # This ID is of the PortfolioStock record itself
+                    stock_id=holding.stock.id,
+                    stock_symbol=holding.stock.symbol,
+                    stock_name=holding.stock.name,
+                    quantity=holding.quantity,
+                    purchase_price=holding.purchase_price,  # Average unit purchase price
+                    purchase_date=holding.purchase_date,  # Date of first/last buy or as defined
+                    current_price=current_price,
+                    market_value=market_value,
+                    gain_loss=gain_loss,
+                    gain_loss_percentage=gain_loss_percentage,
+                    notes=holding.notes,
+                    created_at=holding.created_at,
+                )
+            )
+        return results
+
+    @staticmethod
+    async def _get_utt_holdings_with_values(
+        portfolio_id: int,
+    ) -> List[UTTHoldingResponse]:
+        holdings_orm = (
+            await PortfolioUTT.filter(portfolio_id=portfolio_id)
+            .prefetch_related("utt_fund")
+            .all()
+        )
+        results = []
+        for holding in holdings_orm:  # holding is now an aggregated record
+            latest_nav_data = (
+                await UTTFundData.filter(fund_id=holding.utt_fund_id)
+                .order_by("-date")
+                .first()
+            )
+            current_nav = latest_nav_data.nav_per_unit if latest_nav_data else None
+
+            market_value = (
+                (Decimal(current_nav) * holding.units_held)
+                if current_nav is not None
+                else None
+            )
+            # holding.purchase_price is average unit price
+            cost_basis = holding.purchase_price * holding.units_held
+            gain_loss = (
+                (market_value - cost_basis) if market_value is not None else None
+            )
+            gain_loss_percentage = (
+                (gain_loss / cost_basis * Decimal("100"))
+                if gain_loss is not None and cost_basis > 0
+                else None
+            )
+
+            results.append(
+                UTTHoldingResponse(
+                    id=holding.id,  # This ID is of the PortfolioUTT record itself
+                    utt_fund_id=holding.utt_fund.id,
+                    fund_symbol=holding.utt_fund.symbol,
+                    fund_name=holding.utt_fund.name,
+                    units_held=holding.units_held,
+                    purchase_price=holding.purchase_price,  # Average unit purchase price
+                    purchase_date=holding.purchase_date,  # Date of first/last buy or as defined
+                    current_nav=current_nav,
+                    market_value=market_value,
+                    gain_loss=gain_loss,
+                    gain_loss_percentage=gain_loss_percentage,
+                    notes=holding.notes,
+                    created_at=holding.created_at,
+                )
+            )
+        return results
+
+    @staticmethod
+    async def _get_bond_holdings_with_values(
+        portfolio_id: int,
+    ) -> List[BondHoldingResponse]:
+        holdings_orm = (
+            await PortfolioBond.filter(portfolio_id=portfolio_id)
+            .prefetch_related("bond")
+            .all()
+        )
+        results = []
+        for holding in holdings_orm:  # holding is now an aggregated record
+            current_price_percentage = (
+                holding.bond.price_per_100
+                if hasattr(holding.bond, "price_per_100") and holding.bond.price_per_100
+                else Decimal("100")
+            )
+            market_value = Decimal(
+                holding.face_value_held * current_price_percentage
+            ) / Decimal("100")
+            # print(f"cu")
+            # holding.purchase_price on PortfolioBond model is the TOTAL cost of this aggregated holding
+            cost_basis = holding.purchase_price
+            gain_loss = (
+                (market_value - cost_basis) if market_value is not None else None
+            )
+
+            results.append(
+                BondHoldingResponse(
+                    id=holding.id,  # This ID is of the PortfolioBond record itself
+                    bond_id=holding.bond.id,
+                    instrument_type=holding.bond.instrument_type,
+                    auction_number=(
+                        holding.bond.auction_number
+                        if hasattr(holding.bond, "auction_number")
+                        else None
+                    ),
+                    maturity_date=holding.bond.maturity_date,
+                    face_value_held=holding.face_value_held,
+                    purchase_price=cost_basis,  # Reporting total purchase price of this holding
+                    purchase_date=holding.purchase_date,  # Date of first/last buy or as defined
+                    current_price=current_price_percentage,
+                    market_value=market_value,
+                    accrued_interest=None,
+                    yield_to_maturity=None,
+                    gain_loss=gain_loss,
+                    notes=holding.notes,
+                    created_at=holding.created_at,
+                )
+            )
+        return results
+
+    @staticmethod
+    async def add_stock_to_portfolio(
+        portfolio_id: int,
+        stock_id: int,
+        quantity_to_add: Decimal,  # Quantity for this specific purchase
+        purchase_price_of_lot: Decimal,  # Unit price for this specific purchase
+        purchase_date: date,
+        notes: Optional[str] = None,
+    ) -> PortfolioStock:
+        stock_obj = await Stock.get_or_none(id=stock_id)
+
+        if not stock_obj:
+            raise DoesNotExist("Stock not found")
+        if quantity_to_add <= 0:
+            raise AppException(
+                status_code=400, detail="Quantity to add must be positive."
+            )
+
+        async with in_transaction():
+            holding = await PortfolioStock.get_or_none(
+                portfolio_id=portfolio_id, stock_id=stock_id
+            )
+
+            if holding:
+                # Update existing aggregated holding
+                new_total_cost = (holding.quantity * holding.purchase_price) + (
+                    quantity_to_add * purchase_price_of_lot
+                )
+                holding.quantity += quantity_to_add
+                if holding.quantity > 0:
+                    holding.purchase_price = (
+                        new_total_cost / holding.quantity
+                    )  # New average price
+                else:  # Should not happen if quantity_to_add is positive
+                    holding.purchase_price = purchase_price_of_lot
+
+                holding.purchase_date = purchase_date  # Update to latest purchase_date
+                if notes:
+                    holding.notes = (
+                        f"{holding.notes}\n{notes}".strip() if holding.notes else notes
+                    )
+                await holding.save()
+            else:
+                # Create new holding
+                holding = await PortfolioStock.create(
+                    portfolio_id=portfolio_id,
+                    stock=stock_obj,
+                    quantity=quantity_to_add,
+                    purchase_price=purchase_price_of_lot,  # Initial average price is this lot's price
+                    purchase_date=purchase_date,
+                    notes=notes,
+                )
+
+            await PortfolioTransaction.create(
+                portfolio_id=portfolio_id,
+                transaction_type="BUY",
+                asset_type="STOCK",
+                asset_id=stock_obj.id,
+                asset_name=stock_obj.symbol,
+                quantity=quantity_to_add,
+                price=purchase_price_of_lot,
+                total_amount=quantity_to_add * purchase_price_of_lot,
+                transaction_date=purchase_date,
+                notes=notes or f"Bought {quantity_to_add} shares of {stock_obj.symbol}",
+            )
+        return holding
+
+    @staticmethod
+    async def sell_stock_holding(
+        portfolio_id: int,
+        stock_id: int,  # This is the asset_id
+        quantity_to_sell: Decimal,
+        sell_price: Decimal,
+        sell_date: date,
+        notes: Optional[str] = None,
+    ) -> PortfolioTransaction:
+        # Fetch the stock object to ensure it exists (optional, but good practice)
+        # stock_obj = await Stock.get_or_none(id=stock_id)
+        # if not stock_obj:
+        #     raise DoesNotExist("Stock definition not found.")
+
+        # Fetch the aggregated holding by portfolio_id and stock_id
+        holding = await PortfolioStock.get_or_none(
+            portfolio_id=portfolio_id, stock_id=stock_id
+        ).prefetch_related(
+            "stock"
+        )  # prefetch_related is good if you need stock.symbol etc.
+
+        if not holding:
+            raise DoesNotExist("Stock holding not found in this portfolio.")
+        if quantity_to_sell <= 0:
+            raise AppException(
+                status_code=400, detail="Quantity to sell must be positive."
+            )
+        if holding.quantity < quantity_to_sell:
+            raise AppException(
+                status_code=400,
+                detail=f"Not enough shares to sell. Currently hold {holding.quantity}, trying to sell {quantity_to_sell}.",
+            )
+
+        async with in_transaction():
+            transaction = await PortfolioTransaction.create(
+                portfolio_id=portfolio_id,
+                transaction_type="SELL",
+                asset_type="STOCK",
+                asset_id=holding.stock.id,  # stock_id
+                asset_name=holding.stock.symbol,
+                quantity=quantity_to_sell,
+                price=sell_price,
+                total_amount=quantity_to_sell * sell_price,
+                transaction_date=sell_date,
+                notes=notes
+                or f"Sold {quantity_to_sell} shares of {holding.stock.symbol}",
+            )
+            holding.quantity -= quantity_to_sell
+            # The average purchase_price of the holding does not change upon selling.
+            if holding.quantity == 0:
+                await holding.delete()
+            else:
+                await holding.save()
+        return transaction
+
+    @staticmethod
+    async def add_utt_to_portfolio(
+        portfolio_id: int,
+        utt_fund_id: int,
+        units_to_add: Decimal,  # Units for this specific purchase
+        purchase_price_of_lot: Decimal,  # Unit price for this specific purchase
+        purchase_date: date,
+        notes: Optional[str] = None,
+    ) -> PortfolioUTT:
+        utt_fund_obj = await UTTFund.get_or_none(id=utt_fund_id)
+        if not utt_fund_obj:
+            raise DoesNotExist("UTT Fund not found")
+        if units_to_add <= 0:
+            raise AppException(status_code=400, detail="Units to add must be positive.")
+
+        async with in_transaction():
+            holding = await PortfolioUTT.get_or_none(
+                portfolio_id=portfolio_id, utt_fund_id=utt_fund_id
+            )
+
+            if holding:
+                # Update existing aggregated holding
+                new_total_cost = (holding.units_held * holding.purchase_price) + (
+                    units_to_add * purchase_price_of_lot
+                )
+                holding.units_held += units_to_add
+                if holding.units_held > 0:
+                    holding.purchase_price = (
+                        new_total_cost / holding.units_held
+                    )  # New average price
+                else:
+                    holding.purchase_price = purchase_price_of_lot
+
+                holding.purchase_date = purchase_date  # Update to latest purchase_date
+                if notes:
+                    holding.notes = (
+                        f"{holding.notes}\n{notes}".strip() if holding.notes else notes
+                    )
+                await holding.save()
+            else:
+                # Create new holding
+                holding = await PortfolioUTT.create(
+                    portfolio_id=portfolio_id,
+                    utt_fund=utt_fund_obj,
+                    units_held=units_to_add,
+                    purchase_price=purchase_price_of_lot,  # Initial average price
+                    purchase_date=purchase_date,
+                    notes=notes,
+                )
+
+            await PortfolioTransaction.create(
+                portfolio_id=portfolio_id,
+                transaction_type="BUY",
+                asset_type="UTT",
+                asset_id=utt_fund_obj.id,
+                asset_name=utt_fund_obj.symbol,
+                quantity=units_to_add,
+                price=purchase_price_of_lot,
+                total_amount=units_to_add * purchase_price_of_lot,
+                transaction_date=purchase_date,
+                notes=notes or f"Bought {units_to_add} units of {utt_fund_obj.symbol}",
+            )
+        return holding
+
+    @staticmethod
+    async def sell_utt_holding(
+        portfolio_id: int,
+        utt_fund_id: int,  # Changed from holding_id to asset_id
+        units_to_sell: Decimal,
+        sell_price: Decimal,
+        sell_date: date,
+        notes: Optional[str] = None,
+    ) -> PortfolioTransaction:
+        holding = await PortfolioUTT.get_or_none(
+            portfolio_id=portfolio_id, utt_fund_id=utt_fund_id
+        ).prefetch_related("utt_fund")
+
+        if not holding:
+            raise DoesNotExist("UTT holding not found for this fund in the portfolio.")
+        if units_to_sell <= 0:
+            raise AppException(
+                status_code=400, detail="Units to sell must be positive."
+            )
+        if holding.units_held < units_to_sell:
+            raise AppException(
+                status_code=400,
+                detail=f"Not enough units to sell. Currently hold {holding.units_held}, trying to sell {units_to_sell}.",
+            )
+
+        async with in_transaction():
+            transaction = await PortfolioTransaction.create(
+                portfolio_id=portfolio_id,
+                transaction_type="SELL",
+                asset_type="UTT",
+                asset_id=holding.utt_fund.id,  # This is utt_fund_id
+                asset_name=holding.utt_fund.symbol,
+                quantity=units_to_sell,
+                price=sell_price,
+                total_amount=units_to_sell * sell_price,
+                transaction_date=sell_date,
+                notes=notes
+                or f"Sold {units_to_sell} units of {holding.utt_fund.symbol}",
+            )
+            holding.units_held -= units_to_sell
+            # Average purchase_price of the holding remains unchanged.
+            if holding.units_held == 0:
+                await holding.delete()
+            else:
+                await holding.save()
+        return transaction
+
+    @staticmethod
+    async def add_bond_to_portfolio(
+        portfolio_id: int,
+        bond_id: int,
+        face_value_to_add: Decimal,  # Face value for this specific purchase
+        total_purchase_price_of_lot: Decimal,  # TOTAL purchase price for this face_value_to_add
+        purchase_date: date,
+        notes: Optional[str] = None,
+    ) -> PortfolioBond:
+        bond_obj = await Bond.get_or_none(id=bond_id)
+        if not bond_obj:
+            raise DoesNotExist("Bond not found")
+        if face_value_to_add <= 0:
+            raise AppException(
+                status_code=400, detail="Face value to add must be positive."
+            )
+
+        async with in_transaction():
+            holding = await PortfolioBond.get_or_none(
+                portfolio_id=portfolio_id, bond_id=bond_id
+            )
+
+            if holding:
+                # Update existing aggregated holding
+                holding.face_value_held += face_value_to_add
+                holding.purchase_price += (
+                    total_purchase_price_of_lot  # Add total cost to existing total cost
+                )
+
+                holding.purchase_date = purchase_date  # Update to latest purchase_date
+                if notes:
+                    holding.notes = (
+                        f"{holding.notes}\n{notes}".strip() if holding.notes else notes
+                    )
+                await holding.save()
+            else:
+                # Create new holding
+                holding = await PortfolioBond.create(
+                    portfolio_id=portfolio_id,
+                    bond=bond_obj,
+                    face_value_held=face_value_to_add,
+                    purchase_price=total_purchase_price_of_lot,  # Storing total cost for this initial lot
+                    purchase_date=purchase_date,
+                    notes=notes,
+                )
+
+            unit_price_for_transaction = (
+                total_purchase_price_of_lot / face_value_to_add
+                if face_value_to_add > 0
+                else Decimal("0")
+            )
+            await PortfolioTransaction.create(
+                portfolio_id=portfolio_id,
+                transaction_type="BUY",
+                asset_type="BOND",
+                asset_id=bond_obj.id,
+                asset_name=f"Bond {bond_obj.auction_number or bond_obj.id}",
+                quantity=face_value_to_add,
+                price=unit_price_for_transaction,
+                total_amount=total_purchase_price_of_lot,
+                transaction_date=purchase_date,
+                notes=notes
+                or f"Bought {face_value_to_add} face value of Bond {bond_obj.auction_number or bond_obj.id}",
+            )
+        return holding
+
+    @staticmethod
+    async def sell_bond_holding(
+        portfolio_id: int,
+        bond_id: int,  # Changed from holding_id to asset_id
+        face_value_to_sell: Decimal,
+        sell_price_total: Decimal,  # This is TOTAL proceeds for the face_value_to_sell
+        sell_date: date,
+        notes: Optional[str] = None,
+    ) -> PortfolioTransaction:
+        holding = await PortfolioBond.get_or_none(
+            portfolio_id=portfolio_id, bond_id=bond_id
+        ).prefetch_related("bond")
+
+        if not holding:
+            raise DoesNotExist("Bond holding not found for this bond in the portfolio.")
+        if face_value_to_sell <= 0:
+            raise AppException(
+                status_code=400, detail="Face value to sell must be positive."
+            )
+        if holding.face_value_held < face_value_to_sell:
+            raise AppException(
+                status_code=400,
+                detail=f"Not enough face value to sell. Currently hold {holding.face_value_held}, trying to sell {face_value_to_sell}.",
+            )
+
+        async with in_transaction():
+            unit_sell_price = (
+                sell_price_total / face_value_to_sell
+                if face_value_to_sell > 0
+                else Decimal("0")
+            )
+
+            transaction = await PortfolioTransaction.create(
+                portfolio_id=portfolio_id,
+                transaction_type="SELL",
+                asset_type="BOND",
+                asset_id=holding.bond.id,  # This is bond_id
+                asset_name=f"Bond {holding.bond.auction_number or holding.bond.id}",
+                quantity=face_value_to_sell,
+                price=unit_sell_price,
+                total_amount=sell_price_total,
+                transaction_date=sell_date,
+                notes=notes
+                or f"Sold {face_value_to_sell} face value of Bond {holding.bond.auction_number or holding.bond.id}",
+            )
+
+            original_face_value_held = holding.face_value_held
+            original_total_purchase_price = holding.purchase_price
+
+            holding.face_value_held -= face_value_to_sell
+
+            if holding.face_value_held == Decimal(
+                "0"
+            ):  # Ensure exact zero comparison for Decimal
+                await holding.delete()
+            else:
+                # Update the total purchase_price proportionally for the remaining face_value_held
+                if original_face_value_held > 0:
+                    holding.purchase_price = (
+                        holding.face_value_held / original_face_value_held
+                    ) * original_total_purchase_price
+                else:
+                    holding.purchase_price = Decimal(
+                        "0"
+                    )  # Should not be reached if logic is correct
+                await holding.save()
+        return transaction
+
+    @staticmethod
+    async def remove_holding(
+        portfolio_id: int, asset_type_str: str, asset_id_value: int
+    ) -> bool:
+        """
+        Remove an aggregated holding from portfolio. This is a hard delete.
+        asset_id_value corresponds to stock_id, utt_fund_id, or bond_id.
+        """
+        model_to_delete = None
+        asset_id_field_name = None
+
+        if asset_type_str.upper() == "STOCK":
+            model_to_delete = PortfolioStock
+            asset_id_field_name = "stock_id"
+        elif asset_type_str.upper() == "UTT":
+            model_to_delete = PortfolioUTT
+            asset_id_field_name = "utt_fund_id"
+        elif asset_type_str.upper() == "BOND":
+            model_to_delete = PortfolioBond
+            asset_id_field_name = "bond_id"
+        else:
+            raise AppException(
+                status_code=400, detail=f"Unknown asset type: {asset_type_str}"
+            )
+
+        filter_kwargs = {
+            "portfolio_id": portfolio_id,
+            asset_id_field_name: asset_id_value,
+        }
+        deleted_count = await model_to_delete.filter(**filter_kwargs).delete()
+        return deleted_count > 0
+
+    @staticmethod
+    async def create_portfolio_snapshot(
+        portfolio_id: int, snapshot_date_input: Optional[date] = None
+    ) -> PortfolioSnapshot:
+        """
+        Creates or updates a daily snapshot of portfolio performance for a specific date.
+
+        This function correctly calculates historical values by:
+        1. Determining the holdings that existed in the portfolio on the target_date.
+        2. Fetching the last known market price for each of those holdings as of the target_date.
+        3. Aggregating the values to create a point-in-time snapshot.
+        """
+        target_date: date = date.today()
+        if snapshot_date_input:
+            if isinstance(snapshot_date_input, datetime):
+                target_date = snapshot_date_input.date()
+            else:
+                target_date = snapshot_date_input
+
+        # --- Initialize accumulators ---
+        total_market_value = Decimal("0.0")
+        total_cost_basis = Decimal("0.0")
+        stock_val = Decimal("0.0")
+        bond_val = Decimal("0.0")
+        utt_val = Decimal("0.0")
+
+        # --- 1. Process Stock Holdings ---
+        # Get all stock holdings purchased on or before the target date
+        stock_holdings = await PortfolioStock.filter(
+            portfolio_id=portfolio_id, purchase_date__lte=target_date
+        ).select_related("stock")
+
+        for holding in stock_holdings:
+            # Find the most recent price for this stock on or before the target_date
+            price_data = (
+                await StockPriceData.filter(
+                    stock_id=holding.stock_id, date__lte=target_date
+                )
+                .order_by("-date")
+                .first()
+            )
+
+            if price_data and price_data.closing_price is not None:
+                holding_market_value = (
+                    Decimal(holding.quantity) * price_data.closing_price
+                )
+                stock_val += holding_market_value
+
+            # The cost basis is the sum of purchase prices for all holdings that existed at that time
+            total_cost_basis += holding.purchase_price
+
+        # --- 2. Process UTT Holdings ---
+        utt_holdings = await PortfolioUTT.filter(
+            portfolio_id=portfolio_id, purchase_date__lte=target_date
+        ).select_related("utt_fund")
+
+        for holding in utt_holdings:
+            # Find the most recent NAV for this fund on or before the target_date
+            price_data = (
+                await UTTFundData.filter(
+                    fund_id=holding.utt_fund_id, date__lte=target_date
+                )
+                .order_by("-date")
+                .first()
+            )
+
+            if price_data and price_data.nav_per_unit is not None:
+                # Safely convert float to Decimal
+                holding_market_value = holding.units_held * Decimal(
+                    str(price_data.nav_per_unit)
+                )
+                utt_val += holding_market_value
+
+            total_cost_basis += holding.purchase_price
+
+        # --- 3. Process Bond Holdings ---
+        bond_holdings = await PortfolioBond.filter(
+            portfolio_id=portfolio_id, purchase_date__lte=target_date
+        ).select_related("bond")
+
+        for holding in bond_holdings:
+            # NOTE: Bond valuation is complex. The current `Bond` model does not store historical prices.
+            # A simplified valuation is used here: market value is assumed to be the face value.
+            # For a more advanced system, a separate `BondPriceData` table would be needed.
+            holding_market_value = Decimal(holding.face_value_held)
+            bond_val += holding_market_value
+
+            total_cost_basis += holding.purchase_price
+
+        # --- Aggregate all values ---
+        total_market_value = stock_val + bond_val + utt_val
+        unrealized_gain_loss = total_market_value - total_cost_basis
+
+        # --- Create or Update the snapshot for the target_date ---
+        # This prevents duplicate snapshots if the task runs multiple times.
+        snapshot_datetime = datetime.combine(target_date, datetime.min.time())
+
+        snapshot, created = await PortfolioSnapshot.update_or_create(
+            portfolio_id=portfolio_id,
+            snapshot_date=snapshot_datetime,
+            defaults={
+                "total_value": total_market_value,
+                "stock_value": stock_val,
+                "bond_value": bond_val,
+                "utt_value": utt_val,
+                "cash_value": Decimal("0.0"),  # Assuming cash isn't tracked yet
+                "total_cost": total_cost_basis,
+                "unrealized_gain_loss": unrealized_gain_loss,
+            },
+        )
+
+        if created:
+            print(f"Created snapshot for portfolio {portfolio_id} on {target_date}")
+        else:
+            print(f"Updated snapshot for portfolio {portfolio_id} on {target_date}")
+
+        return snapshot
+
+    @staticmethod
+    async def regenerate_snapshots_task(
+        task_id: int, portfolio_id: int, start_date: date = None
+    ):
+        """
+        A robust background task that generates or regenerates historical portfolio snapshots.
+
+        - If a 'start_date' is provided (e.g., from a back-dated transaction), it will start from there.
+        - If 'start_date' is None, it will intelligently find the date of the very first transaction
+          in the portfolio and start from that point, ensuring all possible data is generated.
+        - It always deletes existing snapshots in the target date range before creating new ones
+          to prevent duplicates and ensure data is fresh.
+        """
+        await ImportTask.filter(id=task_id).update(status="running")
+
+        try:
+            # 1. DETERMINE THE START DATE
+            # If no specific start date is given, find the earliest transaction for this portfolio.
+            if not start_date:
+                first_transaction = (
+                    await PortfolioTransaction.filter(portfolio_id=portfolio_id)
+                    .order_by("transaction_date")
+                    .first()
+                )
+
+                if first_transaction:
+                    start_date = first_transaction.transaction_date
+                    print(
+                        f"[Task {task_id}] No start date provided. Found earliest transaction on {start_date}."
+                    )
+                else:
+                    # If there are no transactions, there's nothing to snapshot.
+                    await ImportTask.filter(id=task_id).update(
+                        status="completed",
+                        details={
+                            "message": "No transactions found in portfolio. Nothing to generate."
+                        },
+                    )
+                    print(
+                        f"[Task {task_id}] No transactions for portfolio {portfolio_id}. Task complete."
+                    )
+                    return
+
+            end_date = date.today()
+            print(
+                f"[Task {task_id}] Starting snapshot generation for portfolio {portfolio_id} from {start_date} to {end_date}"
+            )
+
+            # 2. INVALIDATE: Delete all stale snapshots in the date range to ensure a clean slate.
+            start_datetime = datetime.combine(start_date, datetime.min.time())
+            deleted_count = await PortfolioSnapshot.filter(
+                portfolio_id=portfolio_id, snapshot_date__gte=start_datetime
+            ).delete()
+            print(
+                f"[Task {task_id}] Invalidated and deleted {deleted_count} stale snapshots."
+            )
+
+            # 3. REGENERATE: Loop from the start date to today and recreate each snapshot.
+            def date_range(start, end):
+                # Helper to iterate through a range of dates.
+                for n in range(int((end - start).days) + 1):
+                    yield start + timedelta(n)
+
+            generated_count = 0
+            failed_days = []
+            for single_date in date_range(start_date, end_date):
+                try:
+                    # This calls the other service method responsible for calculating and saving
+                    # a single day's snapshot.
+                    await PortfolioService.create_portfolio_snapshot(
+                        portfolio_id=portfolio_id, snapshot_date_input=single_date
+                    )
+                    print(
+                        f"[Task {task_id}] Successfully generated snapshot for {single_date.isoformat()}"
+                    )
+                    generated_count += 1
+                except Exception as e:
+                    # If one day fails (e.g., missing price data), log it and continue.
+                    failed_days.append(single_date.isoformat())
+                    print(
+                        f"[Task {task_id}] WARNING: Could not generate snapshot for {single_date}: {e}"
+                    )
+
+            # 4. FINALIZE: Update the task with a summary of the operation.
+            summary = {
+                "message": "Snapshot generation complete.",
+                "deleted_stale_snapshots": deleted_count,
+                "new_snapshots_generated": generated_count,
+                "failed_days_count": len(failed_days),
+                "failed_days": failed_days,
+                "date_range": f"{start_date.isoformat()} to {end_date.isoformat()}",
+            }
+            await ImportTask.filter(id=task_id).update(
+                status="completed", details=summary
+            )
+            print(f"[Task {task_id}] Completed successfully. Summary: {summary}")
+
+        except Exception as e:
+            # Catch any fatal error during the task and mark it as failed.
+            await ImportTask.filter(id=task_id).update(
+                status="failed",
+                details={
+                    "error": f"A fatal error occurred during snapshot regeneration: {str(e)}"
+                },
+            )
+            print(f"[Task {task_id}] FAILED with a fatal error: {e}")

App/routers/portfolio/utils.py

@@ -0,0 +1,33 @@
+# Add or ensure these imports are present
+from fastapi import BackgroundTasks
+from .service import PortfolioService  # Ensure service is imported
+from App.routers.tasks.models import ImportTask
+from tortoise.expressions import Q  # For querying JSON fields
+from datetime import date
+
+
+async def trigger_regeneration_if_needed(
+    background_tasks: BackgroundTasks,
+    portfolio_id: int,
+    transaction_date: date,
+    reason: str,
+):
+    """Checks if a transaction is back-dated and queues the regeneration task."""
+    if transaction_date < date.today():
+        task = await ImportTask.create(
+            task_type="portfolio_regeneration",
+            status="pending",
+            details={
+                "portfolio_id": portfolio_id,
+                "reason": reason,
+                "start_date": transaction_date.isoformat(),
+            },
+        )
+        background_tasks.add_task(
+            PortfolioService.regenerate_snapshots_task,
+            task.id,
+            portfolio_id,
+            transaction_date,
+        )
+        return "Holding saved. Historical performance data is being updated in the background."
+    return "Holding saved successfully."

App/routers/stocks/crud.py

@@ -0,0 +1,134 @@
+
+from .models import Stock, StockPriceData
+from datetime import datetime, timedelta
+from tortoise.transactions import in_transaction
+from tortoise.queryset import QuerySet
+from .models import Stock, StockPriceData, Dividend # Import Dividend model
+from typing import List, Dict, Optional
+from datetime import datetime as dt # Alias to avoid conflict with date objects
+from decimal import Decimal, InvalidOperation
+from tortoise.exceptions import DoesNotExist, IntegrityError
+
+
+
+async def create_or_get_stock(symbol: str, name: str):
+    """Create or get existing stock by symbol"""
+    return await Stock.get_or_create(symbol=symbol, defaults={"name": name})
+
+
+async def bulk_insert_price_data(stock, data: list[dict]):
+    """Bulk insert price data for a stock"""
+    records = []
+    for row in data:
+        date = datetime.fromisoformat(row["trade_date"]).date()
+        records.append(
+            StockPriceData(
+                stock=stock,
+                date=date,
+                opening_price=row["opening_price"],
+                closing_price=row["closing_price"],
+                high=row["high"],
+                low=row["low"],
+                volume=row["volume"],
+                turnover=row["turnover"],
+                shares_in_issue=row["shares_in_issue"],
+                market_cap=row["market_cap"]
+            )
+        )
+    
+    async with in_transaction():
+        await StockPriceData.bulk_create(records, ignore_conflicts=True)
+
+
+async def get_stock_price_history(stock_id: int, days: int = 365) -> QuerySet:
+    """Get stock price history as a QuerySet (not materialized list)"""
+    current_date = datetime.now().date()
+    start_date = current_date - timedelta(days=days)
+    
+    # Return QuerySet, not materialized results
+    return StockPriceData.filter(
+        stock_id=stock_id,
+        date__gte=start_date
+    ).order_by("-date")
+
+
+async def get_dividends_by_stock_id(stock_id: int) -> List[Dividend]:
+    """Fetches all dividends for a given stock_id, ordered by ex_dividend_date descending."""
+    return await Dividend.filter(stock_id=stock_id).order_by("-ex_dividend_date").all()
+
+async def bulk_upsert_dividends(stock_id: int, scraped_dividends: List[Dict]) -> List[Dividend]:
+    """
+    Upserts dividend data. If a record with the same stock_id, ex_dividend_date, 
+    dividend_amount and type exists, it updates it. Otherwise, creates a new one.
+    """
+    saved_or_updated_dividends = []
+    for div_data in scraped_dividends:
+        try:
+            ex_date_str = div_data.get("Ex-Dividend Date")
+            pay_date_str = div_data.get("Payment Date")
+
+            # Robust date parsing
+            ex_date = None
+            if ex_date_str:
+                try:
+                    ex_date = dt.strptime(ex_date_str, "%b %d, %Y").date()
+                except ValueError:
+                    print(f"Could not parse ex_dividend_date: {ex_date_str}")
+                    continue # Skip this record
+
+            pay_date = None
+            if pay_date_str:
+                try:
+                    pay_date = dt.strptime(pay_date_str, "%b %d, %Y").date()
+                except ValueError:
+                    print(f"Could not parse payment_date: {pay_date_str}")
+                    # We might still want to save if ex_date and amount are present
+                    # For now, let's require payment_date for simplicity of example
+                    continue
+
+
+            if not ex_date or not pay_date: # Ensure essential dates are present
+                print(f"Skipping dividend due to missing/invalid essential dates: {div_data}")
+                continue
+
+            amount_str = str(div_data.get("Dividend", "0")).replace(',', '')
+            yield_str = str(div_data.get("Yield", "0%")).replace('%', '').replace(',', '')
+
+            dividend_amount = Decimal(amount_str) if amount_str else Decimal("0")
+            
+            yield_percentage = None
+            if yield_str and yield_str.lower() != 'n/a' and yield_str.strip() != '-':
+                 try:
+                    yield_percentage = Decimal(yield_str)
+                 except InvalidOperation:
+                    print(f"Could not parse yield: {yield_str}")
+
+
+            dividend_type = div_data.get("Type")
+            if dividend_type and len(dividend_type) > 50: # Truncate if too long
+                dividend_type = dividend_type[:50]
+            
+            defaults = {
+                "payment_date": pay_date,
+                "yield_percentage": yield_percentage,
+                # ensure updated_at is handled by Tortoise auto_now=True
+            }
+            # Use update_or_create to handle existing records based on unique_together constraint
+            # or a similar logic if unique_together is not fully covering all cases
+            dividend_obj, created = await Dividend.update_or_create(
+                stock_id=stock_id,
+                ex_dividend_date=ex_date,
+                dividend_amount=dividend_amount,
+                dividend_type=dividend_type, # Include type in uniqueness check
+                defaults=defaults
+            )
+            saved_or_updated_dividends.append(dividend_obj)
+
+        except (ValueError, InvalidOperation, TypeError) as e:
+            print(f"Error processing or saving dividend data '{div_data}': {e}")
+            continue
+        except IntegrityError as e: # Handles cases where unique_together might be violated if not using update_or_create
+            print(f"Integrity error for dividend data '{div_data}': {e}. Might be a duplicate.")
+            continue
+            
+    return saved_or_updated_dividends

App/routers/stocks/metrics.py

@@ -0,0 +1,47 @@
+import numpy as np
+from .models import StockPriceData
+
+import numpy as np
+from .models import StockPriceData
+
+async def calculate_metrics(stock):
+    data = await StockPriceData.filter(stock=stock).order_by("date").values(
+        "date", "closing_price", "market_cap", "shares_in_issue", "volume"
+    )
+
+    if not data:
+        return None
+
+    closing_prices = [d["closing_price"] for d in data if d["closing_price"]]
+    market_caps = [d["market_cap"] for d in data if d["market_cap"]]
+    volumes = [d["volume"] for d in data if d["volume"]]
+
+    avg_price = float(np.mean(closing_prices))
+    std_dev = float(np.std(closing_prices))
+    return_pct = ((closing_prices[-1] - closing_prices[0]) / closing_prices[0]) * 100 if closing_prices[0] else 0
+    avg_market_cap = float(np.mean(market_caps)) if market_caps else 0
+    avg_volume = float(np.mean(volumes)) if volumes else 0
+
+    # Standard formulas (placeholders until model supports them)
+    eps = None  # stock.eps (Earnings per share)
+    book_value_per_share = None
+    dividend_per_share = None
+
+    pe_ratio = avg_price / eps if eps else "N/A"
+    earnings_yield = (eps / avg_price) * 100 if eps else "N/A"
+    dividend_yield = (dividend_per_share / avg_price) * 100 if dividend_per_share else "N/A"
+    pb_ratio = avg_price / book_value_per_share if book_value_per_share else "N/A"
+
+    return {
+        "symbol": stock.symbol,
+        "average_price": round(avg_price, 2),
+        "return_percentage": round(return_pct, 2),
+        "volatility": round(std_dev, 2),
+        "average_market_cap": round(avg_market_cap),
+        "average_volume": round(avg_volume),
+        "pe_ratio": pe_ratio,
+        "earnings_yield": earnings_yield,
+        "dividend_yield": dividend_yield,
+        "price_to_book": pb_ratio,
+        "notes": "Add EPS, dividend, and book value to enable full fundamental metrics"
+    }

App/routers/stocks/models.py

@@ -0,0 +1,97 @@
+from tortoise.contrib.pydantic.creator import pydantic_queryset_creator
+from tortoise.models import Model
+from tortoise import fields
+from tortoise.contrib.pydantic import pydantic_model_creator
+from tortoise.queryset import QuerySet
+
+
+class Stock(Model):
+    id = fields.IntField(primary_key=True)
+    symbol = fields.CharField(max_length=10, unique=True)
+    name = fields.CharField(max_length=200)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_now=True)
+
+    @staticmethod
+    async def get_list(data):
+        if type(data) == list(Stock):
+            parser = pydantic_queryset_creator(Stock)
+            return await parser.from_queryset(data)
+
+    async def to_dict(self):
+        if type(self) == Stock:
+            parser = pydantic_model_creator(Stock)
+            return await parser.from_tortoise_orm(self)
+
+    class Meta:
+        table = "stocks"
+
+
+class StockPriceData(Model):
+    id = fields.IntField(primary_key=True)
+    stock = fields.ForeignKeyField("models.Stock", related_name="price_data")
+    date = fields.DateField()
+    opening_price = fields.DecimalField(max_digits=15, decimal_places=2)
+    closing_price = fields.DecimalField(max_digits=15, decimal_places=2)
+    high = fields.DecimalField(max_digits=15, decimal_places=2)
+    low = fields.DecimalField(max_digits=15, decimal_places=2)
+    volume = fields.BigIntField()  # Use BigIntField for large numbers
+    turnover = fields.BigIntField()  # Use BigIntField instead of IntField
+    shares_in_issue = fields.BigIntField()  # Use BigIntField for large numbers
+    market_cap = fields.BigIntField()  # Use BigIntField for large numbers
+    created_at = fields.DatetimeField(auto_now_add=True)
+
+    @staticmethod
+    async def get_list(data):
+        if type(data) == QuerySet:
+            parser = pydantic_queryset_creator(StockPriceData)
+            return await parser.from_queryset(data)
+
+    async def to_dict(self):
+        if type(self) == Model:
+            parser = pydantic_model_creator(StockPriceData)
+            return await parser.from_tortoise_orm(self)
+
+    class Meta:
+        table = "stock_price_data"
+        unique_together = (
+            ("stock", "date"),
+        )  # Prevent duplicate entries for same stock/date
+
+
+class Dividend(Model):
+    id = fields.IntField(primary_key=True)
+    stock = fields.ForeignKeyField("models.Stock", related_name="dividends")
+    ex_dividend_date = fields.DateField()
+    dividend_amount = fields.DecimalField(max_digits=15, decimal_places=4)
+    dividend_type = fields.CharField(max_length=50, null=True, blank=True)
+    payment_date = fields.DateField()
+    yield_percentage = fields.DecimalField(
+        max_digits=10, decimal_places=4, null=True, blank=True
+    )
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_now=True)
+
+    @staticmethod
+    async def get_list(data):
+        if type(data) == QuerySet:
+            parser = pydantic_queryset_creator(Dividend)
+            return await parser.from_queryset(data)
+
+    async def to_dict(self):
+        if type(self) == Model:
+            parser = pydantic_model_creator(Dividend)
+            return await parser.from_tortoise_orm(self)
+        if type(self) == QuerySet:
+            parser = pydantic_queryset_creator(Dividend)
+            return await parser.from_queryset(self)
+
+    class Meta:
+        table = "dividends"
+        ordering = [
+            "-ex_dividend_date",
+            "-payment_date",
+        ]  # Order by dates descending by default
+
+    def __str__(self):
+        return f"{self.stock.symbol} Dividend: {self.dividend_amount} on {self.ex_dividend_date}"

App/routers/stocks/routes.py

@@ -0,0 +1,232 @@
+from fastapi import APIRouter, BackgroundTasks
+from .schemas import DividendResponse, StockResponse, PriceDataResponse
+from .crud import (
+    create_or_get_stock,
+    bulk_insert_price_data,
+    get_stock_price_history,
+    get_dividends_by_stock_id,
+    bulk_upsert_dividends,
+)
+from .service import fetch_dse_stock_data
+from .metrics import calculate_metrics
+from .models import Stock, StockPriceData
+from App.routers.tasks.models import ImportTask
+from App.schemas import ResponseModel
+from typing import Dict, List
+import datetime
+from datetime import datetime, timedelta
+from .models import Dividend
+from .utils import AsyncCurlCffiDividendScraper, run_stock_import_task
+from App.schemas import AppException
+
+router = APIRouter(prefix="/stocks", tags=["stocks"])
+
+
+# --- Caching Mechanism (Simple in-memory for example) ---
+dividend_cache: Dict[str, Dict] = {}
+CACHE_DURATION_MINUTES = 2
+
+
+@router.post("/import/{symbol}", response_model=ResponseModel)
+async def queue_import_stock(symbol: str, background_tasks: BackgroundTasks):
+    task = await ImportTask.create(
+        task_type="stocks", status="pending", details={"symbol": symbol}
+    )
+    background_tasks.add_task(run_stock_import_task, task.id, symbol)
+    return ResponseModel(
+        success=True, message="Stock import task queued", data={"task_id": task.id}
+    )
+
+
+# Alternative Method: Using Tortoise ORM with GROUP BY (if raw SQL not preferred)
+@router.get("/list", response_model=ResponseModel)
+async def list_stocks_orm():
+    """
+    Alternative using Tortoise ORM - less efficient but more ORM-friendly
+    """
+    try:
+        # Get all stocks
+        stocks = await Stock.all()
+
+        if not stocks:
+            raise AppException(status_code=404, detail="No stocks found")
+
+        # Get latest price for each stock in batch
+        stock_ids = [stock.id for stock in stocks]
+
+        # Create a dictionary to store latest prices
+        latest_prices = {}
+
+        # For each stock, get only the latest price (most recent date)
+        for stock_id in stock_ids:
+            latest_price = (
+                await StockPriceData.filter(stock_id=stock_id).order_by("-date").first()
+            )
+
+            if latest_price:
+                latest_prices[stock_id] = latest_price
+
+        # Build the response
+        stock_list = []
+        for stock in stocks:
+            latest = latest_prices.get(stock.id)
+
+            stock_data = {
+                "id": stock.id,
+                "symbol": stock.symbol,
+                "name": stock.name,
+                "latest_price": float(latest.closing_price) if latest else None,
+                "opening_price": float(latest.opening_price) if latest else None,
+                "high": float(latest.high) if latest else None,
+                "low": float(latest.low) if latest else None,
+                "volume": latest.volume if latest else None,
+                "market_cap": latest.market_cap if latest else None,
+                "latest_date": latest.date.isoformat() if latest else None,
+            }
+            stock_list.append(stock_data)
+
+        return ResponseModel(
+            success=True,
+            message=f"Retrieved {len(stock_list)} stocks with latest prices",
+            data={"stocks": stock_list, "count": len(stock_list)},
+        )
+
+    except Exception as e:
+        raise AppException(status_code=500, detail=f"Error retrieving stocks: {str(e)}")
+
+
+@router.get("/{symbol}/prices", response_model=ResponseModel)
+async def get_stock_prices(symbol: str):
+    stock = await Stock.get_or_none(symbol=symbol.upper())
+    if not stock:
+        raise AppException(status_code=404, detail="Stock not found")
+
+    prices = await get_stock_price_history(stock.id)
+    prices_pydantic = await StockPriceData.get_list(prices)
+    return ResponseModel(
+        success=True,
+        message="Stock prices retrieved",
+        data={"prices": prices_pydantic.model_dump()},
+    )
+
+
+@router.get("/{symbol}/metrics", response_model=ResponseModel)
+async def get_stock_metrics(symbol: str):
+    stock = await Stock.get_or_none(symbol=symbol.upper())
+    if not stock:
+        raise AppException(status_code=404, detail="Stock not found")
+
+    metrics = await calculate_metrics(stock)
+    print(
+        metrics
+    )  # Debugging print statement to check the metrics returned by calculate_metrics()
+    if not metrics:
+        raise AppException(
+            status_code=404, detail="No price data available for metrics calculation"
+        )
+
+    return ResponseModel(
+        success=True, message="Stock metrics calculated", data={"metrics": metrics}
+    )
+
+
+# --- New Dividend Route ---
+@router.get("/{symbol}/dividends", response_model=ResponseModel)
+async def get_stock_dividends(symbol: str):
+    stock_symbol_upper = symbol.upper()
+    stock = await Stock.get_or_none(symbol=stock_symbol_upper)
+
+    if not stock:
+        raise AppException(
+            status_code=404,
+            detail=f"Stock '{stock_symbol_upper}' not found and could not be auto-created.",
+        )
+
+    cache_key = f"dividends_{stock_symbol_upper}"
+    current_time = datetime.now()
+
+    # 1. Check cache
+    if cache_key in dividend_cache:
+        cached_item = dividend_cache[cache_key]
+        if current_time - cached_item["timestamp"] < timedelta(
+            minutes=CACHE_DURATION_MINUTES
+        ):
+            print(f"Returning cached dividend data for {stock_symbol_upper}")
+            return ResponseModel(
+                success=True,
+                message="Cached dividend data retrieved",
+                data={"dividends": cached_item["data"]},
+            )
+        else:
+            print(f"Cache expired for {stock_symbol_upper}")
+            del dividend_cache[cache_key]
+
+    # 2. If not in valid cache, try DB
+    db_dividends_orm = await get_dividends_by_stock_id(stock.id)
+
+    # Prepare data for Pydantic response if found in DB
+    if db_dividends_orm:
+        response_data = [
+            DividendResponse.model_validate(div).model_dump(mode="json")
+            for div in db_dividends_orm
+        ]
+        dividend_cache[cache_key] = {"timestamp": current_time, "data": response_data}
+        print(f"Returning dividend data from DB for {stock_symbol_upper}")
+        return ResponseModel(
+            success=True,
+            message="Dividend data retrieved from database",
+            data={"dividends": response_data},
+        )
+
+    # 3. If not in DB (or cache expired and we decide to force refresh, though current logic only scrapes if DB empty)
+    #    Scrape fresh data
+    print(f"No dividends in DB for {stock_symbol_upper}. Scraping fresh data...")
+    scraper = AsyncCurlCffiDividendScraper()
+
+    # The scraper's search_company method implicitly checks for DSE.
+    # We might not need to call _search_company_async separately if fetch_and_extract handles it.
+    # Let's assume fetch_and_extract_dividends_async uses _search_company_async internally to get the URL.
+    scraped_data_list: List[Dict] = await scraper.fetch_and_extract_dividends_async(
+        stock_symbol_upper
+    )
+
+    if not scraped_data_list:
+        # Cache the fact that no data was found to prevent re-scraping immediately
+        dividend_cache[cache_key] = {"timestamp": current_time, "data": []}
+        print(f"Scraper returned no dividend data for {stock_symbol_upper}")
+        return ResponseModel(
+            success=True,
+            message="No dividend data found from scraper",
+            data={"dividends": []},
+        )
+
+    # Save scraped data to DB
+    # bulk_upsert_dividends should convert dicts to ORM objects and save them
+    # and return the list of saved/updated ORM objects.
+    saved_dividends_orm: List[Dividend] = await bulk_upsert_dividends(
+        stock.id, scraped_data_list
+    )
+
+    if not saved_dividends_orm:
+        return ResponseModel(
+            success=False,
+            message="Data scraped but failed to save to DB.",
+            data={"dividends": []},
+        )
+
+    # Convert saved ORM objects to Pydantic response models
+    response_data_after_scrape = [
+        DividendResponse.model_validate(div).model_dump(mode="json")
+        for div in saved_dividends_orm
+    ]
+
+    dividend_cache[cache_key] = {
+        "timestamp": current_time,
+        "data": response_data_after_scrape,
+    }
+    print(f"Successfully scraped and saved dividend data for {stock_symbol_upper}")
+    return ResponseModel(
+        success=True,
+        message="Dividend data scraped and saved",
+        data={"dividends": response_data_after_scrape},
+    )

App/routers/stocks/schemas.py

@@ -0,0 +1,48 @@
+from pydantic import BaseModel
+from datetime import date
+from typing import List
+from datetime import date, datetime as dt # Alias to avoid 
+from decimal import Decimal
+from typing import Optional
+
+
+class DividendBase(BaseModel):
+    ex_dividend_date: date
+    dividend_amount: Decimal
+    dividend_type: Optional[str] = None
+    payment_date: date
+    yield_percentage: Optional[Decimal] = None
+
+    class Config:
+        from_attributes = True # Pydantic v2: use from_attributes = True
+        # orm_mode = True # Pydantic v1
+
+class DividendCreate(DividendBase):
+    pass
+
+class DividendResponse(DividendBase):
+    id: int
+    stock_id: int # Useful to confirm which stock it belongs to
+    created_at: dt
+    updated_at: dt
+
+
+class StockCreate(BaseModel):
+    symbol: str
+    name: str
+
+
+class StockResponse(BaseModel):
+    id: int
+    symbol: str
+    name: str
+
+
+class PriceDataResponse(BaseModel):
+    date: date
+    opening_price: float
+    closing_price: float
+    high: float
+    low: float
+    volume: int
+    turnover: int

App/routers/stocks/service.py

@@ -0,0 +1,9 @@
+import httpx
+
+
+async def fetch_dse_stock_data(symbol: str, days: int = 3000):
+    url = f"https://dse.co.tz/api/get/market/prices/for/range/duration?security_code={symbol}&days={days}&class=EQUITY"
+    print(url)
+    async with httpx.AsyncClient() as client:
+        resp = await client.get(url)
+        return resp.json()

App/routers/stocks/utils.py

@@ -0,0 +1,258 @@
+from datetime import datetime
+from tortoise.transactions import in_transaction
+from App.routers.tasks.models import ImportTask
+from .models import Stock# Updated imports to match used models
+from .service import fetch_dse_stock_data  # Added missing imports
+from .crud import create_or_get_stock, bulk_insert_price_data  # Added missing imports
+
+import asyncio
+from curl_cffi.requests import AsyncSession # For requests-like error handling
+from curl_cffi import CurlError # For lower-level cURL errors
+from bs4 import BeautifulSoup
+import pandas as pd
+import sys # For platform specific asyncio policy
+
+async def run_stock_import_task(task_id: int, symbol: str):
+    try:
+        await ImportTask.filter(id=task_id).update(status="running")
+        data = await fetch_dse_stock_data(symbol)
+      
+        if not data.get("success") or not data.get("data"):
+            await ImportTask.filter(id=task_id).update(status="failed", details={"error": "No data available"})
+            return
+
+        raw = data["data"]
+        first = raw[0]
+        stock, _ = await create_or_get_stock(first["company"], first["fullName"])
+        await bulk_insert_price_data(stock, raw)
+        
+        await ImportTask.filter(id=task_id).update(status="completed")
+    except Exception as e:
+        await ImportTask.filter(id=task_id).update(status="failed", details={"error": str(e)})
+
+
+
+
+class AsyncCurlCffiDividendScraper:
+    BASE_URL = "https://www.investing.com"
+    SEARCH_API_URL = "https://api.investing.com/api/search/v2/search"
+    # Choose an impersonation profile. "chrome" is a good default.
+    # You can also use specific versions like "chrome124"
+    IMPERSONATE_PROFILE = "chrome124" 
+
+    def __init__(self):
+        self.headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', # Example, curl_cffi handles this mostly
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.9',
+            'Domain-Id': 'www' # Crucial for the Investing.com API
+        }
+
+    async def _search_company_async(self, session: AsyncSession, ticker_symbol: str):
+        """
+        Asynchronously searches for a company by its ticker symbol using curl_cffi.
+        Returns its URL suffix if found on the Dar Es Salaam stock exchange.
+        """
+        params = {'q': ticker_symbol}
+        url_suffix = None
+        print(f"Searching for ticker with curl_cffi: {ticker_symbol}...")
+        try:
+            response = await session.get(
+                self.SEARCH_API_URL,
+                params=params,
+                headers=self.headers,
+                impersonate=self.IMPERSONATE_PROFILE,
+                timeout=10
+            )
+
+            response.raise_for_status() 
+            data = response.json()
+
+            if not data or 'quotes' not in data:
+                print(f"No data or quotes found for ticker: {ticker_symbol}")
+                return None
+
+            for quote in data.get('quotes', []):
+                if quote.get('symbol', '').upper() == ticker_symbol.upper() and \
+                   quote.get('exchange', '').lower() == 'dar es salaam':
+                    print(f"Found '{quote.get('description')}' ({quote.get('symbol')}) on {quote.get('exchange')}")
+                    url_suffix = quote.get('url')
+                    break 
+            
+            if not url_suffix:
+                print(f"Ticker {ticker_symbol} not found on Dar Es Salaam stock exchange or no matching symbol.")
+            return url_suffix
+
+        except Exception as e:
+            print(f"HTTP error during async company search for {ticker_symbol}: {e.response.status_code} - {e.response.text[:200]}")
+        except CurlError as e:
+            print(f"cURL error during async company search API call for {ticker_symbol}: {e}")
+        except asyncio.TimeoutError:
+            print(f"Timeout during async company search for {ticker_symbol}")
+        except ValueError as e: 
+            print(f"Error decoding JSON from async search API for {ticker_symbol}: {e}")
+        except Exception as e:
+            print(f"An unexpected error occurred during search for {ticker_symbol}: {e}")
+        return None
+
+
+    async def get_dividend_history_async(self, session: AsyncSession, company_url_suffix: str):
+        """
+        Asynchronously fetches and parses the dividend history table using curl_cffi.
+        """
+        if not company_url_suffix:
+            return []
+
+        if not company_url_suffix.startswith('/'):
+            company_url_suffix = '/' + company_url_suffix
+            
+        dividend_url = f"{self.BASE_URL}{company_url_suffix}-dividends"
+        print(f"Fetching dividend data async with curl_cffi from: {dividend_url}")
+
+        try:
+            response = await session.get(
+                dividend_url,
+                headers=self.headers,
+                impersonate=self.IMPERSONATE_PROFILE,
+                timeout=15
+            )
+            # print(f"Dividend Page Status for {dividend_url}: {response.status_code}") # Debug
+            response.raise_for_status()
+            html_content = response.text # .text is a property in curl_cffi.requests.Response
+        except Exception as e:
+            print(f"HTTP error fetching dividend page {dividend_url}: {e.response.status_code} - {e.response.text[:200]}")
+            return []
+        except CurlError as e:
+            print(f"cURL error fetching dividend page async {dividend_url}: {e}")
+            return []
+        except asyncio.TimeoutError:
+            print(f"Timeout fetching dividend page async {dividend_url}")
+            return []
+        except Exception as e:
+            print(f"An unexpected error occurred fetching dividend page {dividend_url}: {e}")
+            return []
+
+        soup = BeautifulSoup(html_content, 'html.parser')
+        
+        target_table = None
+        # Try to find table by specific class or structure
+        # The original HTML's table has class "freeze-column-w-1 w-full overflow-x-auto text-xs leading-4"
+        # A more reliable way if classes are stable is a more specific selector.
+        # Let's try to find a table that looks like the one in the sample
+        all_tables = soup.find_all('table')
+        for table_candidate in all_tables:
+            # Check for characteristic headers
+            ths = table_candidate.select('thead th')
+            header_texts = [th.get_text(strip=True) for th in ths]
+            if "Ex-Dividend Date" in header_texts and "Dividend" in header_texts and "Payment Date" in header_texts:
+                # Further check: does it have the 'freeze-column-w-1' class which seems distinctive
+                if 'freeze-column-w-1' in table_candidate.get('class', []):
+                    target_table = table_candidate
+                    break
+        
+        if not target_table: # Fallback: if the specific class isn't there, but headers match
+            for table_candidate in all_tables:
+                ths = table_candidate.select('thead th')
+                header_texts = [th.get_text(strip=True) for th in ths]
+                if "Ex-Dividend Date" in header_texts and "Dividend" in header_texts:
+                    target_table = table_candidate
+                    print(f"Found dividend table using fallback (header check) on {dividend_url}")
+                    break
+
+        if not target_table:
+            print(f"Dividend table not found on the page: {dividend_url}")
+            # with open(f"debug_dividend_page_{company_url_suffix.replace('/', '_')}.html", "w", encoding="utf-8") as f:
+            #     f.write(soup.prettify())
+            # print(f"Debug HTML saved for {dividend_url}")
+            return []
+
+        dividends_data = []
+        tbody = target_table.find('tbody')
+        if not tbody:
+            print(f"Table body (tbody) not found in the dividend table on {dividend_url}.")
+            return []
+            
+        rows = tbody.find_all('tr')
+        if not rows:
+            print(f"No rows found in the dividend table body on {dividend_url}.")
+            return []
+
+        for row_idx, row in enumerate(rows):
+            cols = row.find_all('td')
+            if len(cols) >= 5:
+                try:
+                    ex_dividend_date_tag = cols[0].find('time')
+                    ex_dividend_date = ex_dividend_date_tag.get_text(strip=True) if ex_dividend_date_tag else cols[0].get_text(strip=True)
+                    
+                    dividend_val = cols[1].get_text(strip=True)
+                    
+                    type_cell = cols[2]
+                    # The structure for 'Type' in the NMB example has the value in a div after an svg
+                    # e.g., <div class="absolute top-[7px] ...">12M</div>
+                    type_specific_div = type_cell.select_one('div > svg + div') # div containing svg and then the target div
+                    if not type_specific_div: # Try another common pattern from the HTML
+                         type_specific_div = type_cell.select_one('div > div[class*="absolute"]')
+                    
+                    if type_specific_div:
+                        dividend_type = type_specific_div.get_text(strip=True)
+                    else: # Fallback
+                        dividend_type = type_cell.get_text(strip=True).replace('\n', ' ').strip()
+
+                    payment_date_tag = cols[3].find('time')
+                    payment_date = payment_date_tag.get_text(strip=True) if payment_date_tag else cols[3].get_text(strip=True)
+                    
+                    yield_val = cols[4].get_text(strip=True)
+
+                    dividends_data.append({
+                        "Ex-Dividend Date": ex_dividend_date,
+                        "Dividend": dividend_val,
+                        "Type": dividend_type,
+                        "Payment Date": payment_date,
+                        "Yield": yield_val
+                    })
+                except Exception as e:
+                    print(f"Error parsing row {row_idx} on {dividend_url}: {e}. Row content: {[c.get_text(strip=True) for c in cols]}")
+            else:
+                print(f"Skipping row {row_idx} with insufficient columns ({len(cols)}) on {dividend_url}: {[c.get_text(strip=True) for c in cols]}")
+        
+        return dividends_data
+
+    async def fetch_and_extract_dividends_async(self, ticker_symbol: str):
+        """
+        Main async method to search for a company and extract its dividend history.
+        Manages its own curl_cffi.AsyncSession.
+        """
+        async with AsyncSession() as session: # session is created here
+            company_url_suffix = await self._search_company_async(session, ticker_symbol)
+            if company_url_suffix:
+                return await self.get_dividend_history_async(session, company_url_suffix)
+        return []
+
+# --- Example Async Usage ---
+async def main():
+    scraper = AsyncCurlCffiDividendScraper()
+
+    tickers_to_scrape = ["NMB", "VODA"] 
+
+    tasks = [scraper.fetch_and_extract_dividends_async(ticker) for ticker in tickers_to_scrape]
+    
+    results = await asyncio.gather(*tasks, return_exceptions=True)
+
+    for ticker, result in zip(tickers_to_scrape, results):
+        print(f"\n--- Results for {ticker} using curl_cffi ---")
+        if isinstance(result, Exception):
+            print(f"An error occurred while fetching data for {ticker}: {result}")
+        elif result: 
+            if result: 
+                df = pd.DataFrame(result)
+                print(df.to_string())
+            else:
+                print(f"No dividend data extracted for {ticker} (table might be empty or parsing failed after finding the table).")
+        else: 
+            print(f"Could not fetch/extract dividend data for {ticker} (company likely not found on DSE or critical error).")
+
+# if __name__ == "__main__":
+#     if sys.platform == "win32" and sys.version_info >= (3, 8):
+#          asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+            
+#     asyncio.run(main())

App/routers/tasks/models.py

@@ -0,0 +1,19 @@
+from tortoise import fields, models
+from enum import Enum
+
+class TaskStatus(str, Enum):
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+class ImportTask(models.Model):
+    id = fields.IntField(pk=True)
+    task_type = fields.CharField(max_length=50)
+    status = fields.CharEnumField(TaskStatus, default=TaskStatus.PENDING)
+    details = fields.JSONField(null=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_now=True)
+
+    class Meta:
+        table = "import_tasks"

App/routers/tasks/routes.py

@@ -0,0 +1,26 @@
+from fastapi import APIRouter, HTTPException
+from .models import ImportTask
+from .schemas import ImportTaskResponse
+from App.schemas import ResponseModel
+from tortoise.contrib.pydantic import pydantic_queryset_creator, pydantic_model_creator
+router = APIRouter(prefix="/tasks", tags=["Tasks"])
+TaskData_Pydantic_List = pydantic_queryset_creator(
+    ImportTask,
+)
+TaskData_Pydantic = pydantic_model_creator(
+    ImportTask,
+
+)
+@router.get("/", response_model=ResponseModel)
+async def list_tasks():
+    tasks =  ImportTask.all().order_by("-created_at")
+    pydantic_tasks= await TaskData_Pydantic_List.from_queryset(tasks)
+    return ResponseModel(success=True, message="List of tasks", data=pydantic_tasks.model_dump())
+
+@router.get("/{task_id}", response_model=ResponseModel)
+async def get_task(task_id: int):
+    task = await ImportTask.get_or_none(id=task_id)
+    if not task:
+        raise HTTPException(status_code=404, detail="Task not found")
+    pydantic_task = await TaskData_Pydantic.from_tortoise_orm(task)
+    return ResponseModel(success=True, message="Task found", data=pydantic_task.model_dump())

App/routers/tasks/schemas.py

@@ -0,0 +1,23 @@
+from pydantic import BaseModel
+from datetime import datetime
+from enum import Enum
+from pydantic import  ConfigDict
+class TaskStatus(str, Enum):
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+class ImportTaskResponse(BaseModel):
+    id: int
+    task_type: str
+    status: TaskStatus
+    details: dict | None
+    created_at: datetime
+    updated_at: datetime
+
+class ResponseModel(BaseModel):
+    success: bool
+    message: str
+    data: dict | list | None = None
+    model_config = ConfigDict(from_attributes=True)

App/routers/users/models.py

@@ -0,0 +1,42 @@
+from tortoise import fields, models
+from tortoise.contrib.pydantic.creator import pydantic_model_creator, pydantic_queryset_creator
+from tortoise.queryset import QuerySet
+
+class User(models.Model):
+    id = fields.IntField(pk=True)
+    username = fields.CharField(max_length=50, unique=True)
+    email = fields.CharField(max_length=100, unique=True)
+    hashed_password = fields.CharField(max_length=128)
+    created_at = fields.DatetimeField(auto_now_add=True)
+
+    @staticmethod
+    async def get_list(data):
+        if type(data) == QuerySet:
+            parser=pydantic_queryset_creator(User)
+            return await parser.from_queryset(data)
+
+
+
+    async def to_dict(self):
+        if type(self) == User:
+            parser=pydantic_model_creator(User)
+            return await parser.from_tortoise_orm(self)
+
+
+class Watchlist(models.Model):
+    id = fields.IntField(pk=True)
+    user = fields.ForeignKeyField("models.User", related_name="watchlist")
+    stock = fields.ForeignKeyField("models.Stock", null=True, related_name="watching")
+    utt = fields.ForeignKeyField("models.UTTFund", null=True, related_name="watching")
+
+    @staticmethod
+    async def get_list(data):
+        if type(data) == QuerySet:
+            parser=pydantic_queryset_creator(Watchlist)
+            return await parser.from_queryset(data)
+
+
+    async def to_dict(self):
+        if type(self) == models.Model:
+            parser=pydantic_model_creator(Watchlist)
+            return await parser.from_tortoise_orm(self)

App/routers/users/routes.py

@@ -0,0 +1,70 @@
+from fastapi import APIRouter, HTTPException, Depends
+from .models import User, Watchlist
+from App.routers.portfolio.models import Portfolio
+from .schemas import UserCreate, UserResponse, PortfolioItemSchema, WatchlistItemSchema, UserLogin
+from App.schemas import ResponseModel
+from tortoise.contrib.pydantic import pydantic_model_creator, pydantic_queryset_creator
+from passlib.hash import bcrypt
+from App.schemas import AppException
+
+
+router = APIRouter(prefix="/users", tags=["Users"])
+
+User_Pydantic = pydantic_model_creator(User, name="User")
+Portfolio_Pydantic = pydantic_queryset_creator(Portfolio, name="Portfolio")
+Portfolio_one_Pydantic = pydantic_model_creator(Portfolio, name="Portfolio")
+
+@router.post("/login", response_model=ResponseModel)
+async def login(user: UserLogin):
+    user_obj = await User.get_or_none(email=user.email)
+    if not user_obj:
+        raise AppException(status_code=400, detail=ResponseModel(success=False, message="Invalid email or password"))
+    
+    if not bcrypt.verify(user.password, user_obj.hashed_password):
+        raise AppException(status_code=400, detail=ResponseModel(success=False, message="Invalid email or password"))
+    _user = await user_obj.to_dict()
+    _user = UserResponse.model_validate(_user.model_dump())
+    return ResponseModel(success=True, message="Login successful", data=_user)
+
+@router.post("/register", response_model=ResponseModel)
+async def register(user: UserCreate):
+    existing = await User.get_or_none(email=user.email)
+    if existing:
+        raise AppException(status_code=400, detail=ResponseModel(success=False, message="Email already registered"))
+    user_obj = await User.create(
+        username=user.username,
+        email=user.email,
+        hashed_password=bcrypt.hash(user.password)
+    )
+    await Portfolio.create(user=user_obj, name="Default Portfolio")
+    return ResponseModel(success=True, message="User created", data=await User_Pydantic.from_tortoise_orm(user_obj))
+
+@router.get("/{user_id}/portfolio", response_model=ResponseModel)
+async def get_portfolio(user_id: int):
+    portfolios =  Portfolio.filter(user=user_id).all()
+    _portfolios = await Portfolio_Pydantic.from_queryset(portfolios)
+    return ResponseModel(success=True, message="Portfolio retrieved", data=_portfolios.model_dump())
+
+@router.post("/{user_id}/portfolio", response_model=ResponseModel)
+async def create_portfolio(user_id: int, data: PortfolioItemSchema):
+    # Check if user exists
+    user = await User.get_or_none(id=user_id)
+    if not user:
+        raise HTTPException(status_code=404, detail="User not found")
+    
+    # Create a new portfolio
+    portfolio = await Portfolio.create(
+        user=user,
+        name=data.name
+    )
+    _portfolio=Portfolio_one_Pydantic.from_orm(portfolio)    
+    return ResponseModel(success=True, message="Portfolio created", data=_portfolio.model_dump())
+
+@router.post("/{user_id}/watchlist/add", response_model=ResponseModel)
+async def add_to_watchlist(user_id: int, item: WatchlistItemSchema):
+    new_item = await Watchlist.create(
+        user_id=user_id,
+        stock_id=item.stock_id,
+        utt_id=item.utt_id
+    )
+    return ResponseModel(success=True, message="Added to watchlist", data=new_item)

App/routers/users/schemas.py

@@ -0,0 +1,28 @@
+from pydantic import BaseModel, EmailStr
+from typing import Optional
+
+class UserCreate(BaseModel):
+    username: str
+    email: EmailStr
+    password: str
+
+class UserLogin(BaseModel):
+    email: EmailStr
+    password: str
+
+class UserResponse(BaseModel):
+    id: int
+    username: str
+    email: str
+
+class PortfolioItemSchema(BaseModel):
+    name:str
+
+class WatchlistItemSchema(BaseModel):
+    stock_id: Optional[int]
+    utt_id: Optional[int]
+
+class ResponseModel(BaseModel):
+    success: bool
+    message: str
+    data: Optional[dict] = None

App/routers/users/utils.py

@@ -0,0 +1,11 @@
+
+
+from App.schemas import AppException
+from .models import User
+
+async def get_current_user(user_id: int):
+    """Get current user by ID"""
+    user = await User.get_or_none(id=user_id)
+    if not user:
+        raise AppException(status_code=404, detail="User not found")
+    return user  ## can you implement this function

App/routers/utt/models.py

@@ -0,0 +1,48 @@
+from tortoise import fields, models
+from pydantic import BaseModel, ConfigDict
+from tortoise.contrib.pydantic.creator import pydantic_model_creator, pydantic_queryset_creator
+from tortoise.queryset import QuerySet
+
+class UTTFund(models.Model):
+    id = fields.IntField(pk=True)
+    symbol = fields.CharField(max_length=20, unique=True)
+    name = fields.CharField(max_length=100)
+
+    @staticmethod
+    async def get_list(data):
+        if type(data) == type([UTTFund]):
+            parser=pydantic_queryset_creator(UTTFund)
+            return await parser.from_queryset(data)
+
+    async def to_dict(self):
+        if type(self) == UTTFund:
+            parser=pydantic_model_creator(UTTFund)
+            return await parser.from_tortoise_orm(self)
+
+
+class UTTFundData(models.Model):
+    id = fields.IntField(pk=True)
+    fund = fields.ForeignKeyField("models.UTTFund", related_name="data")
+    date = fields.DateField()
+    nav_per_unit = fields.FloatField()
+    sale_price_per_unit = fields.FloatField()
+    repurchase_price_per_unit = fields.FloatField()
+    outstanding_number_of_units = fields.BigIntField()
+    net_asset_value = fields.BigIntField()
+
+    @classmethod
+    async def get_list(data):
+        if type(data) == QuerySet:
+            parser=pydantic_queryset_creator(UTTFundData)
+            return await parser.from_queryset(data)
+
+
+    async def to_dict(self):
+        if type(self) == models.Model:
+            parser=pydantic_model_creator(UTTFundData)
+            return await parser.from_tortoise_orm(self)
+
+
+
+    class Meta:
+        unique_together = ("fund", "date")

App/routers/utt/routes.py

@@ -0,0 +1,97 @@
+from fastapi import APIRouter, BackgroundTasks
+from .models import UTTFund, UTTFundData
+from .schemas import UTTFundResponse, UTTFundListResponse, ResponseModel
+from .service import fetch_all_utt_data, parse_utt_api_row
+from App.routers.tasks.models import ImportTask
+from App.routers.utt.utils import run_utt_import_task
+from App.schemas import AppException
+from tortoise.contrib.pydantic import pydantic_queryset_creator
+
+
+UTTFundData_Pydantic_List = pydantic_queryset_creator(UTTFundData)
+UTTFund_Pydantic_List = pydantic_queryset_creator(UTTFund)
+
+router = APIRouter(prefix="/utt", tags=["UTT"])
+
+
+@router.get("/", response_model=ResponseModel)
+async def list_funds_orm():
+    """
+    Alternative using Tortoise ORM - less efficient but more ORM-friendly
+    """
+    try:
+        # Get all funds
+        funds = await UTTFund.all()
+
+        if not funds:
+            raise AppException(status_code=404, detail="No UTT funds found")
+
+        # Get latest data for each fund
+        fund_ids = [fund.id for fund in funds]
+        latest_data = {}
+
+        # For each fund, get only the latest data (most recent date)
+        for fund_id in fund_ids:
+            latest = await UTTFundData.filter(fund_id=fund_id).order_by("-date").first()
+
+            if latest:
+                latest_data[fund_id] = latest
+
+        # Build the response
+        fund_list = []
+        for fund in funds:
+            latest = latest_data.get(fund.id)
+
+            fund_data = {
+                "id": fund.id,
+                "symbol": fund.symbol,
+                "name": fund.name,
+                "nav_per_unit": latest.nav_per_unit if latest else None,
+                "sale_price_per_unit": latest.sale_price_per_unit if latest else None,
+                "repurchase_price_per_unit": (
+                    latest.repurchase_price_per_unit if latest else None
+                ),
+                "outstanding_number_of_units": (
+                    latest.outstanding_number_of_units if latest else None
+                ),
+                "net_asset_value": latest.net_asset_value if latest else None,
+                "latest_date": latest.date.isoformat() if latest else None,
+            }
+            fund_list.append(fund_data)
+
+        return ResponseModel(
+            success=True,
+            message=f"Retrieved {len(fund_list)} UTT funds with latest data",
+            data={"funds": fund_list, "count": len(fund_list)},
+        )
+
+    except Exception as e:
+        raise AppException(
+            status_code=500, detail=f"Error retrieving UTT funds: {str(e)}"
+        )
+
+
+@router.get("/{symbol}", response_model=ResponseModel)
+async def get_fund_data(symbol: str):
+    fund = await UTTFund.get_or_none(symbol=symbol)
+    if not fund:
+        raise AppException(status_code=404, detail="Fund not found")
+    data_queryset = UTTFundData.filter(fund=fund).order_by("-date").limit(100)
+    utt_fund_data_pydantic = await UTTFundData_Pydantic_List.from_queryset(
+        data_queryset
+    )
+
+    return ResponseModel(
+        success=True,
+        message="Fund data",
+        data={"data": utt_fund_data_pydantic.model_dump()},
+    )
+
+
+@router.post("/import-all", response_model=ResponseModel)
+async def queue_import_utt(background_tasks: BackgroundTasks):
+    task = await ImportTask.create(task_type="utt", status="pending", details={})
+    background_tasks.add_task(run_utt_import_task, task.id)
+    return ResponseModel(
+        success=True, message="UTT import task queued", data={"task_id": task.id}
+    )

App/routers/utt/schemas.py

@@ -0,0 +1,21 @@
+from pydantic import BaseModel
+from datetime import date
+from typing import List
+from App.routers.tasks.schemas import ResponseModel
+
+class UTTFundResponse(BaseModel):
+    id: int
+    symbol: str
+    name: str
+
+class UTTFundDataResponse(BaseModel):
+    date: date
+    nav_per_unit: float
+    sale_price_per_unit: float
+    repurchase_price_per_unit: float
+    outstanding_number_of_units: int
+    net_asset_value: int
+
+class UTTFundListResponse(BaseModel):
+    fund: UTTFundResponse
+    data: List[UTTFundDataResponse]

App/routers/utt/service.py

@@ -0,0 +1,127 @@
+import asyncio
+import re
+from datetime import datetime
+import httpx
+
+async def fetch_all_utt_data():
+    base_headers = {
+        'Accept': 'application/json, text/javascript, */*; q=0.01',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Connection': 'keep-alive',
+        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+        'Origin': 'https://www.uttamis.co.tz',
+        'Referer': 'https://www.uttamis.co.tz/fund-performance',
+        'Sec-Fetch-Dest': 'empty',
+        'Sec-Fetch-Mode': 'cors',
+        'Sec-Fetch-Site': 'same-origin',
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
+        'X-Requested-With': 'XMLHttpRequest',
+        'sec-ch-ua': '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
+        'sec-ch-ua-mobile': '?0',
+        'sec-ch-ua-platform': '"Windows"',
+    }
+
+    all_data = []
+    start = 0
+    length = 4000
+    max_records = 5000
+
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        # Fetch the initial page to get CSRF token and cookies
+        get_response = await client.get("https://www.uttamis.co.tz/fund-performance", headers=base_headers)
+        if get_response.status_code != 200:
+            print(f"Failed to fetch CSRF token: {get_response.status_code}")
+            return []
+        
+        html_content = get_response.text
+        print(html_content[1000:2000])
+        csrf_token_match = re.search(r'<meta name="csrf-token" content="([^"]+)"', html_content)
+        if not csrf_token_match:
+            print("CSRF token not found.")
+            return []
+        csrf_token = csrf_token_match.group(1)
+        
+        # Prepare POST headers with CSRF token
+        post_headers = base_headers.copy()
+        post_headers['X-CSRF-TOKEN'] = csrf_token
+
+        while start < max_records:
+            payload = {
+                "csrf-token": csrf_token,
+                "draw": "1",
+                "start": str(start),
+                "length": str(length),
+                "search[value]": "",
+                "search[regex]": "false",
+            }
+
+            # Columns configuration
+            for i, col in enumerate([
+                ("DT_RowIndex", False),
+                ("sname.name", True),
+                ("net_asset_value", True),
+                ("outstanding_number_of_units", True),
+                ("nav_per_unit", True),
+                ("sale_price_per_unit", True),
+                ("repurchase_price_per_unit", True),
+                ("date_valued", True),
+            ]):
+                data_key = col[0].split(".")[0]
+                payload[f"columns[{i}][data]"] = data_key
+                payload[f"columns[{i}][name]"] = col[0]
+                payload[f"columns[{i}][searchable]"] = str(col[1]).lower()
+                payload[f"columns[{i}][orderable]"] = str(col[1]).lower()
+                payload[f"columns[{i}][search][value]"] = ""
+                payload[f"columns[{i}][search][regex]"] = "false"
+
+            # Make the POST request
+            response = await client.post(
+                "https://www.uttamis.co.tz/navs",
+                data=payload,
+                headers=post_headers
+            )
+
+            if response.status_code != 200:
+                print(f"Request failed at offset {start}: {response.status_code}")
+                break
+
+            json_data = response.json()
+            rows = json_data.get("data", [])
+            if not rows:
+                break
+            all_data.extend(rows)
+
+            # Check if there are more records
+            if len(rows) < length:
+                break
+            start += length
+
+    return all_data
+
+def parse_utt_api_row(row: dict) -> dict:
+    def clean_number(value: str) -> float:
+        return float(value.replace(',', ''))
+    
+    date_str = row.get("date_valued")
+    try:
+        date = datetime.strptime(date_str, "%d-%b-%Y").date()
+    except ValueError:
+        date = datetime.strptime(date_str, "%d-%m-%Y").date()
+    
+    return {
+        "date": date,
+        "nav_per_unit": clean_number(row["nav_per_unit"]),
+        "sale_price_per_unit": clean_number(row["sale_price_per_unit"]),
+        "repurchase_price_per_unit": clean_number(row["repurchase_price_per_unit"]),
+        "outstanding_number_of_units": int(clean_number(row["outstanding_number_of_units"])),
+        "net_asset_value": int(clean_number(row["net_asset_value"]))
+    }
+
+# Example usage
+async def main():
+    data = await fetch_all_utt_data()
+    parsed_data = [parse_utt_api_row(row) for row in data]
+    print(parsed_data)
+
+if __name__ == "__main__":
+    asyncio.run(main())

App/routers/utt/utils.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+from tortoise.transactions import in_transaction
+from App.routers.tasks.models import ImportTask
+from .models import UTTFund, UTTFundData
+from .service import fetch_all_utt_data, parse_utt_api_row
+
+async def run_utt_import_task(task_id: int):
+    try:
+        await ImportTask.filter(id=task_id).update(status="running")
+        raw_data = await fetch_all_utt_data()
+        if not raw_data:
+            await ImportTask.filter(id=task_id).update(status="failed", details={"error": "No data"})
+            return
+
+        for row in raw_data:
+            symbol = row["internal_name"]
+            name = row["scheme_name"]
+            fund, _ = await UTTFund.get_or_create(symbol=symbol, defaults={"name": name})
+
+            parsed = parse_utt_api_row(row)
+            exists = await UTTFundData.exists(fund=fund, date=parsed["date"])
+            if not exists:
+                await UTTFundData.create(fund=fund, **parsed)
+
+        await ImportTask.filter(id=task_id).update(status="completed")
+    except Exception as e:
+        await ImportTask.filter(id=task_id).update(status="failed", details={"error": str(e)})

App/schemas.py

@@ -0,0 +1,25 @@
+from pydantic import BaseModel
+from typing import Optional, Any
+from fastapi import HTTPException
+
+
+class ResponseModel(BaseModel):
+    success: bool
+    message: str
+    data: Optional[Any] = None
+
+
+class AppException(HTTPException):
+    def __init__(self, status_code: int = 400, detail: str | ResponseModel = None):
+        if isinstance(detail, ResponseModel):
+            super().__init__(status_code=status_code, detail=detail.message)
+            self.data = detail.data
+            self.response_model = detail
+        else:
+            super().__init__(status_code=status_code, detail=str(detail) if detail else "An error occurred")
+            self.data = None
+            self.response_model = ResponseModel(
+                success=False,
+                message=str(detail) if detail else "An error occurred",
+                data=None
+            )

Dockerfile

@@ -0,0 +1,77 @@
+# -----------------------------------------------------------------------------
+# Stage 1: Builder Stage
+# This stage installs build-time dependencies and compiles Python packages
+# into wheels, which can be installed in the final stage without build tools.
+# -----------------------------------------------------------------------------
+FROM python:3.11-slim-bullseye AS builder
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /app
+
+# Install system dependencies required for building some of the Python packages
+# (e.g., pandas, curl_cffi)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    libcurl4-openssl-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python dependencies
+COPY requirements.txt .
+RUN pip install --upgrade pip
+# Create a wheelhouse for all dependencies
+RUN pip wheel --no-cache-dir --wheel-dir /wheels -r requirements.txt
+
+
+# -----------------------------------------------------------------------------
+# Stage 2: Final Stage
+# This is the final, lean image. It copies the pre-built wheels from the
+# builder stage and runs the application.
+# -----------------------------------------------------------------------------
+FROM python:3.11-slim-bullseye
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /app
+
+# Install only the runtime system dependencies needed
+# libcurl4 is the runtime library for curl_cffi
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libcurl4 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy the pre-built wheels from the builder stage
+COPY --from=builder /wheels /wheels
+
+# Install the Python dependencies from the wheels
+# This is much faster and doesn't require build tools in the final image
+RUN pip install --no-cache /wheels/*
+
+# Create a non-root user for security
+RUN useradd -m -U -d /home/appuser appuser
+USER appuser
+WORKDIR /home/appuser
+
+# Copy the application code into the container
+# NOTE: The path 'my_app' should match your application's directory name
+COPY --chown=appuser:appuser ./my_app .
+
+# Expose the port the app runs on
+EXPOSE 8000
+
+# --- Database & Migrations Note ---
+# The SQLite DB will be created inside the container.
+# For persistence, mount a volume, e.g., -v ./my_data:/home/appuser/my_data
+# Migrations should be run manually after starting the container.
+# Example: docker exec <container_name> aerich upgrade
+
+# Command to run the application
+# Assumes your main file is `main.py` and your FastAPI instance is `app`.
+# Change `main:app` if your file/variable names are different.
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

db.py

@@ -0,0 +1,43 @@
+from tortoise import Tortoise
+
+DATABASE_URL = "sqlite://db.sqlite3"
+
+TORTOISE_ORM = {
+    "connections": {"default": DATABASE_URL},
+    "apps": {
+        "models": {
+            "models": [
+                'App.routers.stocks.models',
+                'App.routers.tasks.models',
+                'App.routers.utt.models',
+                'App.routers.users.models',
+                'App.routers.portfolio.models',
+                'App.routers.bonds.models',
+                 "aerich.models"
+            ],
+            "default_connection": "default",
+        }
+    }
+}
+
+async def init_db():
+    await Tortoise.init(
+        db_url=DATABASE_URL,
+        modules={'models': [
+            'App.routers.stocks.models',
+            'App.routers.tasks.models',
+            'App.routers.utt.models',
+            'App.routers.users.models',
+            'App.routers.portfolio.models',
+            'App.routers.bonds.models'
+        ]}
+    )
+    await Tortoise.generate_schemas()
+
+async def close_db():
+    await Tortoise.close_connections()
+
+
+async def clear_db():
+    for model in Tortoise.apps.get('models').values():
+        await model.all().delete()

main.py

@@ -0,0 +1,73 @@
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from fastapi.exceptions import RequestValidationError, HTTPException
+from starlette.status import HTTP_400_BAD_REQUEST
+from App.routers.stocks.routes import router as stocks_router
+from App.routers.utt.routes import router as utt_router
+from App.routers.bonds.routes import router as bonds_router
+from App.routers.tasks.routes import router as tasks_router
+from App.routers.users.routes import router as users_router
+from App.routers.portfolio.routes import router as portfolio_router
+from App.schemas import ResponseModel,AppException
+
+from db import init_db, close_db,clear_db
+
+app = FastAPI(title="Uwekezaji API", description="Stock Market Data API")
+
+
+
+@app.exception_handler(AppException)
+async def custom_http_exception_handler(request: Request, exc: AppException):
+    return JSONResponse(
+        status_code=exc.status_code,
+        content=ResponseModel(
+            success=False,
+            message=exc.detail,
+            data=exc.data
+        ).dict()
+    )
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request: Request, exc: RequestValidationError):
+    return JSONResponse(
+        status_code=HTTP_400_BAD_REQUEST,
+        content=ResponseModel(
+            success=False,
+            message="Validation error",
+            data={"errors": exc.errors()}
+        ).dict()
+    )
+
+# Configure CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Include routers
+app.include_router(stocks_router)
+app.include_router(utt_router)
+app.include_router(bonds_router)
+app.include_router(tasks_router)
+app.include_router(users_router)
+app.include_router(portfolio_router)
+
+# Database initialization and cleanup
+@app.on_event("startup")
+async def startup_event():
+    # Clear and reinitialize database on startup
+    await init_db()
+
+@app.on_event("shutdown")
+async def shutdown_event():
+    # await clear_db()
+    await close_db()
+
+# Root endpoint
+@app.get("/")
+async def root():
+    return {"message": "Welcome to Uwekezaji API"}

migrations/models/0_20250525140513_init.py

@@ -0,0 +1,174 @@
+from tortoise import BaseDBAsyncClient
+
+
+async def upgrade(db: BaseDBAsyncClient) -> str:
+    return """
+        CREATE TABLE IF NOT EXISTS "stocks" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "symbol" VARCHAR(10) NOT NULL UNIQUE,
+    "name" VARCHAR(200) NOT NULL,
+    "created_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP
+);
+CREATE TABLE IF NOT EXISTS "stock_price_data" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "date" DATE NOT NULL,
+    "opening_price" VARCHAR(40) NOT NULL,
+    "closing_price" VARCHAR(40) NOT NULL,
+    "high" VARCHAR(40) NOT NULL,
+    "low" VARCHAR(40) NOT NULL,
+    "volume" BIGINT NOT NULL,
+    "turnover" BIGINT NOT NULL,
+    "shares_in_issue" BIGINT NOT NULL,
+    "market_cap" BIGINT NOT NULL,
+    "created_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "stock_id" INT NOT NULL REFERENCES "stocks" ("id") ON DELETE CASCADE,
+    CONSTRAINT "uid_stock_price_stock_i_1f3075" UNIQUE ("stock_id", "date")
+);
+CREATE TABLE IF NOT EXISTS "import_tasks" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "task_type" VARCHAR(50) NOT NULL,
+    "status" VARCHAR(9) NOT NULL  DEFAULT 'pending' /* PENDING: pending\nRUNNING: running\nCOMPLETED: completed\nFAILED: failed */,
+    "details" JSON,
+    "created_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP
+);
+CREATE TABLE IF NOT EXISTS "uttfund" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "symbol" VARCHAR(20) NOT NULL UNIQUE,
+    "name" VARCHAR(100) NOT NULL
+);
+CREATE TABLE IF NOT EXISTS "uttfunddata" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "date" DATE NOT NULL,
+    "nav_per_unit" REAL NOT NULL,
+    "sale_price_per_unit" REAL NOT NULL,
+    "repurchase_price_per_unit" REAL NOT NULL,
+    "outstanding_number_of_units" BIGINT NOT NULL,
+    "net_asset_value" BIGINT NOT NULL,
+    "fund_id" INT NOT NULL REFERENCES "uttfund" ("id") ON DELETE CASCADE,
+    CONSTRAINT "uid_uttfunddata_fund_id_4fe3c3" UNIQUE ("fund_id", "date")
+);
+CREATE TABLE IF NOT EXISTS "user" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "username" VARCHAR(50) NOT NULL UNIQUE,
+    "email" VARCHAR(100) NOT NULL UNIQUE,
+    "hashed_password" VARCHAR(128) NOT NULL,
+    "created_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP
+);
+CREATE TABLE IF NOT EXISTS "watchlist" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "stock_id" INT REFERENCES "stocks" ("id") ON DELETE CASCADE,
+    "user_id" INT NOT NULL REFERENCES "user" ("id") ON DELETE CASCADE,
+    "utt_id" INT REFERENCES "uttfund" ("id") ON DELETE CASCADE
+);
+CREATE TABLE IF NOT EXISTS "portfolios" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "name" VARCHAR(100) NOT NULL,
+    "description" TEXT,
+    "is_active" INT NOT NULL  DEFAULT 1,
+    "created_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "user_id" INT NOT NULL REFERENCES "user" ("id") ON DELETE CASCADE,
+    CONSTRAINT "uid_portfolios_user_id_d03ed2" UNIQUE ("user_id", "name")
+);
+CREATE TABLE IF NOT EXISTS "portfolio_calendar" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "event_date" DATE NOT NULL,
+    "event_type" VARCHAR(50) NOT NULL,
+    "title" VARCHAR(200) NOT NULL,
+    "description" TEXT,
+    "asset_type" VARCHAR(10),
+    "asset_id" INT,
+    "estimated_amount" VARCHAR(40),
+    "is_completed" INT NOT NULL  DEFAULT 0,
+    "created_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "portfolio_id" INT NOT NULL REFERENCES "portfolios" ("id") ON DELETE CASCADE
+);
+CREATE TABLE IF NOT EXISTS "portfolio_snapshots" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "snapshot_date" DATE NOT NULL,
+    "total_value" VARCHAR(40) NOT NULL,
+    "stock_value" VARCHAR(40) NOT NULL  DEFAULT 0,
+    "bond_value" VARCHAR(40) NOT NULL  DEFAULT 0,
+    "utt_value" VARCHAR(40) NOT NULL  DEFAULT 0,
+    "cash_value" VARCHAR(40) NOT NULL  DEFAULT 0,
+    "total_cost" VARCHAR(40) NOT NULL,
+    "unrealized_gain_loss" VARCHAR(40) NOT NULL,
+    "created_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "portfolio_id" INT NOT NULL REFERENCES "portfolios" ("id") ON DELETE CASCADE,
+    CONSTRAINT "uid_portfolio_s_portfol_dc81b0" UNIQUE ("portfolio_id", "snapshot_date")
+) /* Daily snapshots for performance tracking */;
+CREATE TABLE IF NOT EXISTS "portfolio_stocks" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "quantity" INT NOT NULL,
+    "purchase_price" VARCHAR(40) NOT NULL,
+    "purchase_date" DATE NOT NULL,
+    "notes" TEXT,
+    "created_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "portfolio_id" INT NOT NULL REFERENCES "portfolios" ("id") ON DELETE CASCADE,
+    "stock_id" INT NOT NULL REFERENCES "stocks" ("id") ON DELETE CASCADE
+);
+CREATE TABLE IF NOT EXISTS "portfolio_transactions" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "transaction_type" VARCHAR(20) NOT NULL,
+    "asset_type" VARCHAR(10) NOT NULL,
+    "asset_id" INT NOT NULL,
+    "quantity" VARCHAR(40) NOT NULL,
+    "price" VARCHAR(40) NOT NULL,
+    "total_amount" VARCHAR(40) NOT NULL,
+    "transaction_date" DATE NOT NULL,
+    "notes" TEXT,
+    "created_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "portfolio_id" INT NOT NULL REFERENCES "portfolios" ("id") ON DELETE CASCADE
+) /* Track all portfolio transactions for audit and reporting */;
+CREATE TABLE IF NOT EXISTS "portfolio_utts" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "units_held" VARCHAR(40) NOT NULL,
+    "purchase_price" VARCHAR(40) NOT NULL,
+    "purchase_date" DATE NOT NULL,
+    "notes" TEXT,
+    "created_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "portfolio_id" INT NOT NULL REFERENCES "portfolios" ("id") ON DELETE CASCADE,
+    "utt_fund_id" INT NOT NULL REFERENCES "uttfund" ("id") ON DELETE CASCADE
+);
+CREATE TABLE IF NOT EXISTS "bonds" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "instrument_type" VARCHAR(50) NOT NULL,
+    "auction_number" INT NOT NULL,
+    "auction_date" DATE NOT NULL,
+    "maturity_years" VARCHAR(10) NOT NULL,
+    "maturity_date" DATE NOT NULL,
+    "effective_date" DATE NOT NULL,
+    "dtm" INT NOT NULL,
+    "bond_auction_number" INT NOT NULL,
+    "holding_number" INT NOT NULL,
+    "face_value" BIGINT NOT NULL,
+    "price_per_100" REAL NOT NULL,
+    "coupon_rate" REAL NOT NULL,
+    CONSTRAINT "uid_bonds_auction_2fb1f0" UNIQUE ("auction_number", "auction_date")
+);
+CREATE TABLE IF NOT EXISTS "portfolio_bonds" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "face_value_held" BIGINT NOT NULL,
+    "purchase_price" VARCHAR(40) NOT NULL,
+    "purchase_date" DATE NOT NULL,
+    "notes" TEXT,
+    "created_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP NOT NULL  DEFAULT CURRENT_TIMESTAMP,
+    "bond_id" INT NOT NULL REFERENCES "bonds" ("id") ON DELETE CASCADE,
+    "portfolio_id" INT NOT NULL REFERENCES "portfolios" ("id") ON DELETE CASCADE
+);
+CREATE TABLE IF NOT EXISTS "aerich" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "version" VARCHAR(255) NOT NULL,
+    "app" VARCHAR(100) NOT NULL,
+    "content" JSON NOT NULL
+);"""
+
+
+async def downgrade(db: BaseDBAsyncClient) -> str:
+    return """
+        """

pyproject.toml

@@ -0,0 +1,4 @@
+[tool.aerich]
+tortoise_orm = "db.TORTOISE_ORM"
+location = "./migrations"
+src_folder = "./."

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+pythonpath = .
+addopts = -v

readme.md

@@ -0,0 +1,8 @@
+---
+title: XYHLF
+emoji: 🚀
+colorFrom: purple
+colorTo: gray
+sdk: docker
+app_port: 7860
+---

requirements.txt

@@ -0,0 +1,25 @@
+# Core FastAPI Framework
+fastapi==0.105.0
+uvicorn==0.21.1
+starlette==0.27.0
+pydantic==2.10.2
+
+# Database (Asyncio ORM and SQLite Driver)
+tortoise-orm==0.21.7
+aiosqlite==0.18.0
+
+# Database Migrations
+aerich==0.8.0
+
+# Web Scraping & Data Handling
+curl_cffi==0.11.1
+pandas==2.1.4
+
+# Key Dependencies
+anyio==3.7.1
+click==8.1.7
+h11==0.14.0
+numpy==1.26.4
+python-dateutil==2.9.0.post0
+pytz==2023.3.post1
+typing_extensions==4.12.2

structure.txt

@@ -0,0 +1,388 @@
+project/
+├── main.py
+├── routers/
+│   ├── stocks/
+│   │   ├── __init__.py
+│   │   ├── models.py
+│   │   ├── schemas.py
+│   │   ├── service.py
+│   │   └── routes.py
+│   ├── utt/
+│   │   ├── __init__.py
+│   │   ├── models.py
+│   │   ├── schemas.py
+│   │   ├── service.py
+│   │   └── routes.py
+│   └── tasks/
+│       ├── __init__.py
+│       ├── models.py
+│       ├── schemas.py
+│       └── routes.py
+
+# main.py
+from fastapi import FastAPI, Request, HTTPException
+from fastapi.responses import JSONResponse
+fromApp.routers.stocks.routes import router as stocks_router
+fromApp.routers.utt.routes import router as utt_router
+fromApp.routers.tasks.routes import router as tasks_router
+
+app = FastAPI()
+
+@app.exception_handler(HTTPException)
+async def http_exception_handler(request: Request, exc: HTTPException):
+    return JSONResponse(status_code=exc.status_code, content={"success": False, "message": exc.detail})
+
+@app.exception_handler(Exception)
+async def generic_exception_handler(request: Request, exc: Exception):
+    return JSONResponse(status_code=500, content={"success": False, "message": str(exc)})
+
+app.include_router(stocks_router)
+app.include_router(utt_router)
+app.include_router(tasks_router)
+
+# routers/tasks/models.py
+from tortoise import fields, models
+from enum import Enum
+
+class TaskStatus(str, Enum):
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+class ImportTask(models.Model):
+    id = fields.IntField(pk=True)
+    task_type = fields.CharField(max_length=50)
+    status = fields.CharEnumField(TaskStatus, default=TaskStatus.PENDING)
+    details = fields.JSONField(null=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_now=True)
+
+    class Meta:
+        table = "import_tasks"
+
+# routers/tasks/schemas.py
+from pydantic import BaseModel
+from datetime import datetime
+from enum import Enum
+
+class TaskStatus(str, Enum):
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+class ImportTaskResponse(BaseModel):
+    id: int
+    task_type: str
+    status: TaskStatus
+    details: dict | None
+    created_at: datetime
+    updated_at: datetime
+
+class ResponseModel(BaseModel):
+    success: bool
+    message: str
+    data: dict | list | None = None
+
+# routers/tasks/routes.py
+from fastapi import APIRouter, HTTPException
+from .models import ImportTask
+from .schemas import ImportTaskResponse, ResponseModel
+
+router = APIRouter(prefix="/tasks", tags=["Tasks"])
+
+@router.get("/", response_model=ResponseModel)
+async def list_tasks():
+    tasks = await ImportTask.all().order_by("-created_at")
+    return ResponseModel(success=True, message="List of tasks", data=[task for task in tasks])
+
+@router.get("/{task_id}", response_model=ResponseModel)
+async def get_task(task_id: int):
+    task = await ImportTask.get_or_none(id=task_id)
+    if not task:
+        raise HTTPException(status_code=404, detail="Task not found")
+    return ResponseModel(success=True, message="Task found", data=task)
+
+# You would follow the same modular structure for stocks and utt
+# Including their own models.py, schemas.py, routes.py, service.py
+# The routes would queue background tasks and use task_id for status tracking.
+
+
+
+project/
+├── main.py
+├── routers/
+│   ├── stocks/
+│   │   ├── __init__.py
+│   │   ├── models.py
+│   │   ├── schemas.py
+│   │   ├── service.py
+│   │   └── routes.py
+│   ├── utt/
+│   │   ├── __init__.py
+│   │   ├── models.py
+│   │   ├── schemas.py
+│   │   ├── service.py
+│   │   └── routes.py
+│   └── tasks/
+│       ├── __init__.py
+│       ├── models.py
+│       ├── schemas.py
+│       └── routes.py
+
+# routers/stocks/models.py
+from tortoise import fields, models
+
+class Stock(models.Model):
+    id = fields.IntField(pk=True)
+    symbol = fields.CharField(max_length=10, unique=True)
+    name = fields.CharField(max_length=100)
+    sector = fields.CharField(max_length=100, null=True)
+
+class StockPriceData(models.Model):
+    id = fields.IntField(pk=True)
+    stock = fields.ForeignKeyField("models.Stock", related_name="prices")
+    date = fields.DateField()
+    opening_price = fields.FloatField()
+    closing_price = fields.FloatField()
+    high = fields.FloatField()
+    low = fields.FloatField()
+    volume = fields.IntField()
+    turnover = fields.BigIntField()
+    shares_in_issue = fields.BigIntField()
+    market_cap = fields.BigIntField()
+
+    class Meta:
+        unique_together = ("stock", "date")
+
+# routers/stocks/schemas.py
+from pydantic import BaseModel
+from datetime import date
+from typing import List, Optional
+fromApp.routers.tasks.schemas import ResponseModel
+
+class StockBase(BaseModel):
+    symbol: str
+    name: str
+    sector: Optional[str] = None
+
+class StockResponse(StockBase):
+    id: int
+
+class StockPriceResponse(BaseModel):
+    date: date
+    opening_price: float
+    closing_price: float
+    high: float
+    low: float
+    volume: int
+    turnover: int
+    shares_in_issue: int
+    market_cap: int
+
+class StockPriceListResponse(BaseModel):
+    stock: StockResponse
+    prices: List[StockPriceResponse]
+
+# routers/stocks/service.py
+from datetime import datetime
+
+async def fetch_stock_data(symbol: str):
+    import httpx
+    url = f"https://dse.co.tz/api/get/market/prices/for/range/duration?security_code={symbol}&days=5000&class=EQUITY"
+    async with httpx.AsyncClient() as client:
+        response = await client.get(url)
+        if response.status_code == 200:
+            return response.json().get("data", [])
+        return []
+
+def parse_stock_api_row(row: dict) -> dict:
+    return {
+        "date": datetime.fromisoformat(row["trade_date"]).date(),
+        "opening_price": row["opening_price"],
+        "closing_price": row["closing_price"],
+        "high": row["high"],
+        "low": row["low"],
+        "volume": row["volume"],
+        "turnover": row["turnover"],
+        "shares_in_issue": row["shares_in_issue"],
+        "market_cap": row["market_cap"],
+    }
+
+# routers/stocks/routes.py
+from fastapi import APIRouter, HTTPException, BackgroundTasks
+from tortoise.transactions import in_transaction
+from .models import Stock, StockPriceData
+from .schemas import StockResponse, StockPriceListResponse, ResponseModel
+from .service import fetch_stock_data, parse_stock_api_row
+fromApp.routers.tasks.models import ImportTask
+from typing import List
+
+router = APIRouter(prefix="/stocks", tags=["Stocks"])
+
+@router.get("/", response_model=ResponseModel)
+async def list_stocks():
+    stocks = await Stock.all()
+    return ResponseModel(success=True, message="List of stocks", data=stocks)
+
+@router.get("/{symbol}", response_model=ResponseModel)
+async def get_stock_prices(symbol: str):
+    stock = await Stock.get_or_none(symbol=symbol)
+    if not stock:
+        raise HTTPException(status_code=404, detail="Stock not found")
+    prices = await StockPriceData.filter(stock=stock).order_by("-date")
+    return ResponseModel(success=True, message="Stock price data", data={"stock": stock, "prices": prices})
+
+@router.post("/import/{symbol}", response_model=ResponseModel)
+async def queue_import_stock(symbol: str, background_tasks: BackgroundTasks):
+    task = await ImportTask.create(task_type="stocks", status="pending", details={"symbol": symbol})
+    background_tasks.add_task(run_stock_import_task, task.id, symbol)
+    return ResponseModel(success=True, message="Stock import task queued", data={"task_id": task.id})
+
+async def run_stock_import_task(task_id: int, symbol: str):
+    try:
+        await ImportTask.filter(id=task_id).update(status="running")
+        raw_data = await fetch_stock_data(symbol)
+        if not raw_data:
+            await ImportTask.filter(id=task_id).update(status="failed", details={"error": "No data"})
+            return
+
+        first = raw_data[0]
+        stock, _ = await Stock.get_or_create(symbol=first["company"], defaults={"name": first["fullName"]})
+
+        existing_dates = set(await StockPriceData.filter(stock=stock).values_list("date", flat=True))
+
+        records = []
+        for row in raw_data:
+            parsed = parse_stock_api_row(row)
+            if parsed["date"] not in existing_dates:
+                records.append(StockPriceData(stock=stock, **parsed))
+
+        async with in_transaction():
+            await StockPriceData.bulk_create(records, ignore_conflicts=True)
+
+        await ImportTask.filter(id=task_id).update(status="completed")
+    except Exception as e:
+        await ImportTask.filter(id=task_id).update(status="failed", details={"error": str(e)})
+
+
+# routers/utt/models.py
+from tortoise import fields, models
+
+class UTTFund(models.Model):
+    id = fields.IntField(pk=True)
+    symbol = fields.CharField(max_length=20, unique=True)
+    name = fields.CharField(max_length=100)
+
+class UTTFundData(models.Model):
+    id = fields.IntField(pk=True)
+    fund = fields.ForeignKeyField("models.UTTFund", related_name="data")
+    date = fields.DateField()
+    nav_per_unit = fields.FloatField()
+    sale_price_per_unit = fields.FloatField()
+    repurchase_price_per_unit = fields.FloatField()
+    outstanding_number_of_units = fields.BigIntField()
+    net_asset_value = fields.BigIntField()
+
+    class Meta:
+        unique_together = ("fund", "date")
+
+# routers/utt/schemas.py
+from pydantic import BaseModel
+from datetime import date
+from typing import List
+fromApp.routers.tasks.schemas import ResponseModel
+
+class UTTFundResponse(BaseModel):
+    id: int
+    symbol: str
+    name: str
+
+class UTTFundDataResponse(BaseModel):
+    date: date
+    nav_per_unit: float
+    sale_price_per_unit: float
+    repurchase_price_per_unit: float
+    outstanding_number_of_units: int
+    net_asset_value: int
+
+class UTTFundListResponse(BaseModel):
+    fund: UTTFundResponse
+    data: List[UTTFundDataResponse]
+
+# routers/utt/service.py
+from datetime import datetime
+import httpx
+
+async def fetch_all_utt_data():
+    url = "https://example.com/utt/api"  # Placeholder
+    async with httpx.AsyncClient() as client:
+        response = await client.get(url)
+        if response.status_code == 200:
+            return response.json().get("data", [])
+        return []
+
+def parse_utt_api_row(row: dict) -> dict:
+    return {
+        "date": datetime.strptime(row["date_valued"], "%d-%m-%Y").date(),
+        "nav_per_unit": float(row["nav_per_unit"]),
+        "sale_price_per_unit": float(row["sale_price_per_unit"]),
+        "repurchase_price_per_unit": float(row["repurchase_price_per_unit"]),
+        "outstanding_number_of_units": int(float(row["outstanding_number_of_units"])),
+        "net_asset_value": int(float(row["net_asset_value"]))
+    }
+
+# routers/utt/routes.py
+from fastapi import APIRouter, BackgroundTasks, HTTPException
+from .models import UTTFund, UTTFundData
+from .schemas import UTTFundResponse, UTTFundListResponse, ResponseModel
+from .service import fetch_all_utt_data, parse_utt_api_row
+fromApp.routers.tasks.models import ImportTask
+
+router = APIRouter(prefix="/utt", tags=["UTT"])
+
+@router.get("/", response_model=ResponseModel)
+async def list_funds():
+    funds = await UTTFund.all()
+    return ResponseModel(success=True, message="List of UTT funds", data=funds)
+
+@router.get("/{symbol}", response_model=ResponseModel)
+async def get_fund_data(symbol: str):
+    fund = await UTTFund.get_or_none(symbol=symbol)
+    if not fund:
+        raise HTTPException(status_code=404, detail="Fund not found")
+    data = await UTTFundData.filter(fund=fund).order_by("-date")
+    return ResponseModel(success=True, message="Fund data", data={"fund": fund, "data": data})
+
+@router.post("/import-all", response_model=ResponseModel)
+async def queue_import_utt(background_tasks: BackgroundTasks):
+    task = await ImportTask.create(task_type="utt", status="pending", details={})
+    background_tasks.add_task(run_utt_import_task, task.id)
+    return ResponseModel(success=True, message="UTT import task queued", data={"task_id": task.id})
+
+async def run_utt_import_task(task_id: int):
+    from tortoise.transactions import in_transaction
+    try:
+        await ImportTask.filter(id=task_id).update(status="running")
+        raw_data = await fetch_all_utt_data()
+        if not raw_data:
+            await ImportTask.filter(id=task_id).update(status="failed", details={"error": "No data"})
+            return
+
+        for row in raw_data:
+            symbol = row["internal_name"]
+            name = row["scheme_name"]
+            fund, _ = await UTTFund.get_or_create(symbol=symbol, defaults={"name": name})
+
+            parsed = parse_utt_api_row(row)
+            exists = await UTTFundData.exists(fund=fund, date=parsed["date"])
+            if not exists:
+                await UTTFundData.create(fund=fund, **parsed)
+
+        await ImportTask.filter(id=task_id).update(status="completed")
+    except Exception as e:
+        await ImportTask.filter(id=task_id).update(status="failed", details={"error": str(e)})
+
+
+

tests/conftest.py

@@ -0,0 +1,45 @@
+import pytest
+from tortoise import Tortoise
+
+@pytest.fixture(scope="session")
+async def initialize_tests(request):
+    """Initialize test database"""
+    db_config = {
+        'connections': {
+            'default': {
+                'engine': 'tortoise.backends.sqlite',
+                'credentials': {
+                    'file_path': ':memory:',
+                }
+            },
+        },
+        'apps': {
+            'models': {
+                'models': [
+                    'App.routers.stocks.models',
+                    'App.routers.tasks.models',
+                    'App.routers.utt.models',
+                    'App.routers.users.models',
+                    'App.routers.portfolio.models',
+                    'App.routers.bonds.models'
+                ],
+                'default_connection': 'default',
+            }
+        }
+    }
+    
+    await Tortoise.init(config=db_config)
+    await Tortoise.generate_schemas()
+    
+    yield
+    
+    await Tortoise.close_connections()
+
+@pytest.fixture
+async def client():
+    """Create a test client"""
+    from httpx import AsyncClient
+    from main import app
+    
+    async with AsyncClient(app=app, base_url="http://test") as client:
+        yield client

tests/test_portfolio.py

@@ -0,0 +1,193 @@
+import pytest
+import httpx
+pytest_plugins = ["pytest_asyncio"]
+
+@pytest.mark.asyncio
+async def test_create_portfolio(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        # Test successful creation
+        response = await async_client.post("http://localhost:8001/portfolio/", json={
+            "name": "Test Portfolio",
+            "description": "Test portfolio description"
+        }, params={"user_id":33 })
+        
+        assert response.status_code == 200
+        data = response.json()
+        assert data["success"] == True
+        assert data["message"] == "Portfolio created successfully"
+        assert data["data"]["name"] == "Test Portfolio"
+        assert data["data"]["description"] == "Test portfolio description"
+        assert "id" in data["data"]
+        assert "user_id" in data["data"]
+
+@pytest.mark.asyncio
+async def test_get_portfolio(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        # Create a portfolio first
+        create_response = await async_client.post("http://localhost:8001/portfolio/", json={
+            "name": "Test Portfolio"
+        }, params={"user_id": 33})
+        portfolio_id = create_response.json()["data"]["id"]
+        
+        # Test successful retrieval
+        response = await async_client.get(f"http://localhost:8001/portfolio/{portfolio_id}")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["success"] == True
+        assert data["message"] == "Portfolio retrieved successfully"
+        
+        # Test not found
+        response = await async_client.get("http://localhost:8001/portfolio/99999")
+        assert response.status_code == 404
+        data = response.json()
+        assert data["success"] == False
+        assert "Portfolio not found" in data["detail"]
+
+@pytest.mark.asyncio
+async def test_add_stock(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        # Create portfolio first
+        create_response = await async_client.post("http://localhost:8001/portfolio/", json={
+            "name": "Test Portfolio"
+        }, params={"user_id": 33})
+        portfolio_id = create_response.json()["data"]["id"]
+        
+        # Import a test stock
+        await async_client.post("http://localhost:8001/stocks/import/CRDB")
+        
+        # Test successful addition
+        response = await async_client.post(f"http://localhost:8001/portfolio/{portfolio_id}/stocks", json={
+            "stock_id": 1,
+            "quantity": 100,
+            "purchase_price": 1000,
+            "purchase_date": "2023-01-01"
+        })
+        
+        assert response.status_code == 200
+        data = response.json()
+        assert data["success"] == True
+        assert data["message"] == "Stock added to portfolio successfully"
+        assert data["data"]["quantity"] == 100
+        
+        # Test invalid portfolio
+        response = await async_client.post("http://localhost:8001/portfolio/99999/stocks", json={
+            "stock_id": 1,
+            "quantity": 100,
+            "purchase_price": 1000,
+            "purchase_date": "2023-01-01"
+        })
+        assert response.status_code == 404
+        assert "Portfolio not found" in response.json()["detail"]
+        
+        # Test invalid stock
+        response = await async_client.post(f"http://localhost:8001/portfolio/{portfolio_id}/stocks", json={
+            "stock_id": 99999,
+            "quantity": 100,
+            "purchase_price": 1000,
+            "purchase_date": "2023-01-01"
+        })
+        assert response.status_code == 404
+        assert "Stock not found" in response.json()["detail"]
+
+@pytest.mark.asyncio
+async def test_add_utt(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        # Create portfolio first
+        create_response = await async_client.post("http://localhost:8001/portfolio/", json={
+            "name": "Test Portfolio"
+        }, params={"user_id": 33})
+        portfolio_id = create_response.json()["data"]["id"]
+        
+        # Import UTT funds
+        await async_client.post("http://localhost:8001/utt/import-all")
+        
+        # Test successful addition
+        response = await async_client.post(f"http://localhost:8001/portfolio/{portfolio_id}/utts", json={
+            "utt_fund_id": 1,
+            "units": 100,
+            "purchase_price": 1000,
+            "purchase_date": "2023-01-01"
+        })
+        
+        assert response.status_code == 200
+        data = response.json()
+        assert data["success"] == True
+        assert data["message"] == "UTT added to portfolio successfully"
+        assert data["data"]["units"] == 100
+        assert "id" in data["data"]
+
+@pytest.mark.asyncio
+async def test_add_bond(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        # Create portfolio first
+        create_response = await async_client.post("http://localhost:8001/portfolio/", json={
+            "name": "Test Portfolio"
+        }, params={"user_id": 33})
+        portfolio_id = create_response.json()["data"]["id"]
+        
+        # Import bonds
+        await async_client.post("http://localhost:8001/bonds/import-all")
+        
+        # Test successful addition
+        response = await async_client.post(f"http://localhost:8001/portfolio/{portfolio_id}/bonds", json={
+            "bond_id": 1,
+            "face_value": 10000,
+            "purchase_date": "2023-01-01",
+            "maturity_date": "2024-01-01"
+        })
+        
+        assert response.status_code == 200
+        data = response.json()
+        assert "face_value" in data
+        assert data["face_value"] == 10000
+
+@pytest.mark.asyncio
+async def test_add_calendar_event(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        # Create portfolio first
+        create_response = await async_client.post("http://localhost:8001/portfolio/", json={
+            "name": "Test Portfolio"
+        }, params={"user_id": 33})
+        portfolio_id = create_response.json()["data"]["id"]
+        
+        # Test successful addition
+        response = await async_client.post(f"http://localhost:8001/portfolio/{portfolio_id}/calendar", json={
+            "title": "Test Event",
+            "description": "Test event description",
+            "event_date": "2023-01-01",
+            "event_type": "dividend"
+        })
+        
+        assert response.status_code == 200
+        data = response.json()
+        assert "title" in data
+        assert data["title"] == "Test Event"
+
+@pytest.mark.asyncio
+async def test_remove_items(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        # Create portfolio and add items first
+        create_response = await async_client.post("http://localhost:8001/portfolio/", json={
+            "name": "Test Portfolio"
+        }, params={"user_id": 33})
+        portfolio_id = create_response.json()["data"]["id"]
+        
+        # Add a stock first
+        await async_client.post("http://localhost:8001/stocks/import/CRDB")
+        stock_response = await async_client.post(f"http://localhost:8001/portfolio/{portfolio_id}/stocks", json={
+            "stock_id": 1,
+            "quantity": 100,
+            "purchase_price": 1000,
+            "purchase_date": "2023-01-01"
+        })
+        stock_id = stock_response.json()["data"]["id"]
+        
+        # Test remove stock
+        response = await async_client.delete(f"http://localhost:8001/portfolio/{portfolio_id}/stocks/{stock_id}")
+        assert response.status_code == 200
+        assert response.json()["message"] == "Stock removed from portfolio"
+        
+        # Test remove with invalid IDs
+        response = await async_client.delete(f"http://localhost:8001/portfolio/{portfolio_id}/stocks/99999")
+        assert response.status_code == 404
+        assert "Stock not found in portfolio" in response.json()["detail"]

tests/test_stocks.py

@@ -0,0 +1,51 @@
+import pytest
+import httpx
+pytest_plugins = ["pytest_asyncio"]
+
+@pytest.mark.asyncio
+async def test_queue_import_stock(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        response = await async_client.post("http://localhost:8001/stocks/import/CRDB")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["success"] == True
+        assert data["message"] == "Stock import task queued"
+        assert "task_id" in data["data"]
+
+@pytest.mark.asyncio
+async def test_get_stock_prices_not_found(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        response = await async_client.get("http://localhost:8001/stocks/INVALID/prices")
+        assert response.status_code == 404
+        data = response.json()
+        assert data["success"] == False
+        assert "Stock not found" in data["message"]
+
+@pytest.mark.asyncio
+async def test_get_stock_prices(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        # First import a stock
+        await async_client.post("http://localhost:8001/stocks/import/CRDB")
+        
+        # Then get its prices
+        response = await async_client.get("http://localhost:8001/stocks/CRDB/prices")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["success"] == True
+        assert data["message"] == "Stock prices retrieved"
+        assert "prices" in data["data"]
+
+@pytest.mark.asyncio
+async def test_get_stock_metrics(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        # First import a stock
+        await async_client.post("http://localhost:8001/stocks/import/CRDB")
+        
+        # Then get its metrics
+        response = await async_client.get("http://localhost:8001/stocks/CRDB/metrics")
+        assert response.status_code == 200
+        data = response.json()
+        print(data)
+        assert data["success"] == True
+        assert data["message"] == "Stock metrics calculated"
+        assert "metrics" in data["data"]

tests/test_users.py

@@ -0,0 +1,69 @@
+import pytest
+import httpx
+pytest_plugins = ["pytest_asyncio"]
+
+@pytest.mark.asyncio
+async def test_register_user(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        response = await async_client.post("http://localhost:8001/users/register", json={
+            "username": "testwuser",
+            "email": "test@example.com",
+            "password": "testpassword123"
+        })
+        
+        assert response.status_code == 200
+        data = response.json()
+        assert data["success"] == True
+        assert data["data"]["username"] == "testuser"
+        assert data["data"]["email"] == "test@example.com"
+
+@pytest.mark.asyncio
+async def test_register_duplicate_email(client, initialize_tests):
+    # First registration
+    async with httpx.AsyncClient() as async_client:
+        await async_client.post("http://localhost:8001/users/register", json={
+            "username": "existing",
+            "email": "existing@example.com",
+            "password": "password123"
+        })
+        
+        # Attempt duplicate registration
+        response = await async_client.post("http://localhost:8001/users/register", json={
+            "username": "testwuser",
+            "email": "existing@example.com",
+            "password": "testpassword123"
+        })
+        
+        assert response.status_code == 400
+        data = response.json()
+        print(data)
+        assert data["success"] == False
+        assert "Email already registered" in data["message"]
+
+
+@pytest.mark.asyncio
+async def test_get_portfolio(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        # First create a user
+        register_response = await async_client.post("http://localhost:8001/users/register", json={
+            "username": "portfoliouser",
+            "email": "portfolio@example.com",
+            "password": "password123"
+        })
+        user_id = register_response.json()["data"]["id"]
+        
+        # Create a portfolio for the user
+        portfolio_response = await async_client.post(f"http://localhost:8001/users/{user_id}/portfolio", json={
+            "name": "Test Portfolio"
+        })
+        assert portfolio_response.status_code == 200
+        assert portfolio_response.json()["success"] == True
+        
+        # Get the portfolio
+        get_response = await async_client.get(f"http://localhost:8001/users/{user_id}/portfolio")
+        assert get_response.status_code == 200
+        data = get_response.json()
+        assert data["success"] == True
+        assert "data" in data
+
+

tests/test_utt.py

@@ -0,0 +1,48 @@
+import pytest
+import httpx
+pytest_plugins = ["pytest_asyncio"]
+
+# @pytest.mark.asyncio
+# async def test_list_funds(client, initialize_tests):
+#     async with httpx.AsyncClient() as async_client:
+#         # Import funds first
+#         await async_client.post("http://localhost:8001/utt/import-all")
+        
+#         response = await async_client.get("http://localhost:8001/utt/")
+#         assert response.status_code == 200
+#         data = response.json()
+#         assert data["success"] == True
+#         assert "data" in data
+#         assert len(data["data"]) > 0
+
+@pytest.mark.asyncio
+async def test_get_fund_data(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        # Import funds first
+        # await async_client.post("http://localhost:8001/utt/import-all")
+        
+        # Get specific fund data
+        response = await async_client.get("http://localhost:8001/utt/umoja")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["success"] == True
+        assert data["data"]["fund"]["symbol"] == "umoja"
+        assert "data" in data["data"]
+
+@pytest.mark.asyncio
+async def test_get_fund_data_not_found(client, initialize_tests):
+    async with httpx.AsyncClient() as async_client:
+        response = await async_client.get("http://localhost:8001/utt/INVALID")
+        assert response.status_code == 404
+        data = response.json()
+        assert data["success"] == False
+        assert "Fund not found" in data["message"]
+
+# @pytest.mark.asyncio
+# async def test_queue_import_utt(client, initialize_tests):
+#     async with httpx.AsyncClient() as async_client:
+#         response = await async_client.post("http://localhost:8001/utt/import-all")
+#         assert response.status_code == 200
+#         data = response.json()
+#         assert data["success"] == True
+#         assert "task_id" in data["data"]

vercel.json

@@ -0,0 +1,15 @@
+{
+    "builds": [
+      {
+        "src": "main.py",
+        "use": "@vercel/python"
+      }
+    ],
+    "routes": [
+      {
+        "src": "/(.*)",
+        "dest": "main.py"
+      }
+    ]
+  }
+