app/load_data.py

import os
import glob
import json
import csv
from datetime import datetime

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
DATA_DIR = os.path.join(BASE_DIR, "data")
HISTORICAL_DIR = os.path.join(DATA_DIR, "simulation_historical_data")
PRICE_DIR = os.path.join(DATA_DIR, "simulation_price_data_July_1-Aug_30")
NEWS_DIR = os.path.join(DATA_DIR, "simulation_news_data_July_1-Aug_30")

def extract_symbol(filename):
    name = os.path.basename(filename)
    name = name.replace(".csv", "").replace(".json", "")
    for prefix in ["simulated_"]:
        if name.startswith(prefix):
            name = name[len(prefix):]
    for suffix in ["_2026_historical", "_historical", "_price_data", "_live"]:
        if name.endswith(suffix):
            name = name[:-len(suffix)]
    import re
    match = re.match(r'^[A-Za-z]+', name)
    return match.group(0).upper() if match else name.upper()

def load_historical_csvs():
    historical = {}
    if os.path.exists(HISTORICAL_DIR):
        for filepath in glob.glob(os.path.join(HISTORICAL_DIR, "*.csv")):
            sym = extract_symbol(filepath)
            bars = []
            with open(filepath, "r", encoding="utf-8") as f:
                reader = csv.DictReader(f)
                for row in reader:
                    try:
                        c = float(row["close"])
                        bars.append({
                            "date": row["timestamp"].strip(),
                            "open": round(float(row["open"]), 2),
                            "high": round(float(row["high"]), 2),
                            "low": round(float(row["low"]), 2),
                            "close": round(c, 2),
                            "volume": int(row.get("volume", 0))
                        })
                    except (ValueError, KeyError):
                        continue
            if bars:
                # Compute ma20, ma50, rsi
                for i, b in enumerate(bars):
                    if i >= 19:
                        slice_c = [x["close"] for x in bars[i-19:i+1]]
                        b["ma20"] = round(sum(slice_c) / 20.0, 2)
                    else:
                        b["ma20"] = None
                    if i >= 49:
                        slice_c = [x["close"] for x in bars[i-49:i+1]]
                        b["ma50"] = round(sum(slice_c) / 50.0, 2)
                    else:
                        b["ma50"] = None
                    if i >= 14:
                        gains = []
                        losses = []
                        for j in range(i-13, i+1):
                            prev = bars[j-1]["close"] if j > 0 else bars[j]["close"]
                            diff = bars[j]["close"] - prev
                            if diff > 0:
                                gains.append(diff)
                            else:
                                losses.append(abs(diff))
                        avg_gain = sum(gains) / 14.0
                        avg_loss = sum(losses) / 14.0
                        b["rsi"] = 100.0 if avg_loss == 0 else round(100.0 - (100.0 / (1.0 + (avg_gain / avg_loss))), 1)
                    else:
                        b["rsi"] = None
                historical[sym] = bars
    return historical

def load_price_csvs():
    prices = {}
    if os.path.exists(PRICE_DIR):
        for filepath in glob.glob(os.path.join(PRICE_DIR, "*.csv")):
            sym = extract_symbol(filepath)
            ticks = []
            with open(filepath, "r", encoding="utf-8") as f:
                reader = csv.DictReader(f)
                for row in reader:
                    try:
                        o = float(row["open"])
                        c = float(row["close"])
                        ticks.append({
                            "date": row["timestamp"].strip(),
                            "open": round(o, 2),
                            "high": round(float(row["high"]), 2),
                            "low": round(float(row["low"]), 2),
                            "close": round(c, 2),
                            "volume": int(row.get("volume", 0)),
                            "vwap": round((o + c) / 2.0, 2)
                        })
                    except (ValueError, KeyError):
                        continue
            if ticks:
                prices[sym] = ticks
    return prices

def load_news_jsons():
    news_items = []
    if os.path.exists(NEWS_DIR):
        id_counter = 1
        for filepath in glob.glob(os.path.join(NEWS_DIR, "*.json")):
            try:
                with open(filepath, "r", encoding="utf-8") as f:
                    data = json.load(f)
                    items = []
                    if isinstance(data, list):
                        items = data
                    elif isinstance(data, dict):
                        if "feed" in data and isinstance(data["feed"], list):
                            items = data["feed"]
                        else:
                            for val in data.values():
                                if isinstance(val, list):
                                    items.extend(val)

                    for item in items:
                            syms = [ts.get("ticker") for ts in item.get("ticker_sentiment", []) if ts.get("ticker")]
                            time_pub = item.get("time_published", "")
                            if len(time_pub) == 15 and "T" in time_pub:
                                time_pub = f"{time_pub[:4]}-{time_pub[4:6]}-{time_pub[6:8]}T{time_pub[9:11]}:{time_pub[11:13]}:{time_pub[13:15]}Z"
                            
                            sent_obj = item.get("ticker_sentiment", [{}])[0] if item.get("ticker_sentiment") else {}
                            sent_label = sent_obj.get("ticker_sentiment_label", "")
                            sentiment = "bullish" if "bullish" in sent_label.lower() else ("bearish" if "bearish" in sent_label.lower() else "neutral")
                            
                            raw_score = float(sent_obj.get("ticker_sentiment_score", 0))
                            rel_score = float(sent_obj.get("relevance_score", 0.8))
                            conf_pct = min(99, max(65, int((abs(raw_score) * 0.5 + rel_score * 0.5) * 100)))

                            topics = item.get("topics", [])
                            cat = topics[0].get("topic", "General") if topics else "General"

                            news_items.append({
                                "id": f"news-backend-{id_counter}",
                                "headline": item.get("title", "Market Update"),
                                "summary": item.get("summary", item.get("title", "")),
                                "source": item.get("source", "MarketWatch"),
                                "timestamp": time_pub,
                                "symbols": syms if syms else ["AAPL"],
                                "sentiment": sentiment,
                                "confidence": f"{conf_pct}%",
                                "category": cat
                            })
                            id_counter += 1
            except Exception as e:
                print(f"Error reading news JSON {filepath}: {e}")
    return news_items

def load_all_csv_data():
    return {
        "historical": load_historical_csvs(),
        "prices": load_price_csvs(),
        "news": load_news_jsons()
    }

if __name__ == "__main__":
    data = load_all_csv_data()
    print(f"Loaded {len(data['historical'])} historical symbols, {len(data['prices'])} price symbols, {len(data['news'])} news items.")