app.py

import streamlit as st
import requests
import pandas as pd
import time
import json
from datetime import datetime, timedelta
import asyncio
import aiohttp
from typing import Dict, Any, List
import xml.etree.ElementTree as ET
import io
import sys

# Configuration
st.set_page_config(
    page_title="🇸🇪 API Data Fetcher",
    page_icon="🇸🇪",
    layout="wide",
    initial_sidebar_state="expanded"
)

# Real API Configuration
API_CONFIG = {
    "Skolverket": {
        "base_url": "https://api.skolverket.se",
        "endpoints": {
            "planned-educations": "/planned-educations/v3/compact-school-units",
            "skolenhetsregister": "/skolenhetsregister/v2/skolenhet",
            "syllabus": "/syllabus/v1/studievag"
        },
        "headers": {"accept": "application/json"},
        "rate_limit": None
    },
    "SCB": {
        "base_url": "https://api.scb.se",
        "endpoints": {
            "befolkning": "/OV0104/v1/doris/sv/ssd/BE/BE0101/BE0101A/BefolkningNy"
        },
        "headers": {"accept": "application/json"},
        "rate_limit": {"requests": 10, "per_seconds": 10}
    },
    "Kolada": {
        "base_url": "https://api.kolada.se",
        "endpoints": {
            "kpi": "/v2/kpi",
            "data": "/v2/data/kpi",
            "municipality": "/v2/municipality"
        },
        "headers": {"accept": "application/json"},
        "rate_limit": None
    },
    "Världsbanken": {
        "base_url": "https://api.worldbank.org",
        "endpoints": {
            "indicators": "/v2/country/{country}/indicator/{indicator}",
            "countries": "/v2/country"
        },
        "headers": {"accept": "application/json"},
        "rate_limit": None
    },
    "Riksbanken": {
        "base_url": "https://api.riksbank.se",
        "endpoints": {
            "observations": "/swea/v1/Observations/{series}/{from_date}/{to_date}"
        },
        "headers": {"accept": "application/json"},
        "rate_limit": None
    },
    "CSN": {
        "base_url": "https://statistik.csn.se",
        "endpoints": {
            "studiemedel": "/PXWeb/api/v1/sv/CSNstat/Studiestod/Studiemedel/Hogskola/{table}.px"
        },
        "headers": {"accept": "application/json"},
        "rate_limit": None
    }
}

class RealDataFetcher:
    def __init__(self):
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'API-Data-Fetcher/1.0 (Educational Purpose)'
        })
        
    def fetch_skolverket_data(self, endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]:
        """Fetch real data from Skolverket API"""
        try:
            config = API_CONFIG["Skolverket"]
            url = config["base_url"] + config["endpoints"][endpoint]
            
            # Set proper headers for Skolverket
            headers = config["headers"].copy()
            if endpoint == "planned-educations":
                headers["accept"] = "application/vnd.skolverket.plannededucations.api.v3.hal+json"
            
            response = self.session.get(url, params=params, headers=headers, timeout=30)
            response.raise_for_status()
            
            data = response.json()
            
            # Extract data based on endpoint
            if endpoint == "planned-educations":
                school_units = data.get("_embedded", {}).get("compactSchoolUnits", [])
                return {
                    "status": "success",
                    "data": school_units,
                    "total": len(school_units),
                    "timestamp": datetime.now().isoformat(),
                    "source": "Skolverket"
                }
            elif endpoint == "skolenhetsregister":
                school_units = data.get("Skolenheter", [])
                return {
                    "status": "success", 
                    "data": school_units,
                    "total": len(school_units),
                    "timestamp": datetime.now().isoformat(),
                    "source": "Skolverket"
                }
            else:
                return {
                    "status": "success",
                    "data": data,
                    "timestamp": datetime.now().isoformat(),
                    "source": "Skolverket"
                }
                
        except Exception as e:
            return {
                "status": "error",
                "error": str(e),
                "timestamp": datetime.now().isoformat(),
                "source": "Skolverket"
            }
    
    def fetch_scb_data(self, query: str) -> Dict[str, Any]:
        """Fetch real data from SCB API"""
        try:
            config = API_CONFIG["SCB"]
            url = config["base_url"] + config["endpoints"]["befolkning"]
            
            # Parse query if it's JSON string
            if isinstance(query, str):
                try:
                    query_data = json.loads(query)
                except:
                    query_data = {"query": [], "response": {"format": "json"}}
            else:
                query_data = query
            
            response = self.session.post(url, json=query_data, headers=config["headers"], timeout=30)
            response.raise_for_status()
            
            data = response.json()
            
            return {
                "status": "success",
                "data": data.get("data", []),
                "columns": data.get("columns", []),
                "timestamp": datetime.now().isoformat(),
                "source": "SCB"
            }
            
        except Exception as e:
            return {
                "status": "error",
                "error": str(e),
                "timestamp": datetime.now().isoformat(),
                "source": "SCB"
            }
    
    def fetch_kolada_data(self, kpi: str = None, municipality: str = None, year: int = None) -> Dict[str, Any]:
        """Fetch real data from Kolada API"""
        try:
            config = API_CONFIG["Kolada"]
            
            if kpi:
                # Fetch KPI data
                url = config["base_url"] + config["endpoints"]["data"] + f"/{kpi}"
                params = {}
                if municipality:
                    params["municipality"] = municipality
                if year:
                    params["year"] = year
            else:
                # Fetch municipality list
                url = config["base_url"] + config["endpoints"]["municipality"]
                params = {}
            
            response = self.session.get(url, params=params, headers=config["headers"], timeout=30)
            response.raise_for_status()
            
            data = response.json()
            
            return {
                "status": "success",
                "data": data.get("values", []),
                "timestamp": datetime.now().isoformat(),
                "source": "Kolada"
            }
            
        except Exception as e:
            return {
                "status": "error",
                "error": str(e),
                "timestamp": datetime.now().isoformat(),
                "source": "Kolada"
            }
    
    def fetch_worldbank_data(self, country: str, indicator: str, year: int) -> Dict[str, Any]:
        """Fetch real data from World Bank API"""
        try:
            config = API_CONFIG["Världsbanken"]
            url = config["base_url"] + config["endpoints"]["indicators"].format(
                country=country, indicator=indicator
            )
            
            params = {
                "format": "json",
                "date": str(year)
            }
            
            response = self.session.get(url, params=params, headers=config["headers"], timeout=30)
            response.raise_for_status()
            
            data = response.json()
            
            # World Bank returns [metadata, data] array
            if len(data) > 1 and data[1]:
                result_data = data[1]
            else:
                result_data = []
            
            return {
                "status": "success",
                "data": result_data,
                "timestamp": datetime.now().isoformat(),
                "source": "Världsbanken"
            }
            
        except Exception as e:
            return {
                "status": "error",
                "error": str(e),
                "timestamp": datetime.now().isoformat(),
                "source": "Världsbanken"
            }
    
    def fetch_riksbank_data(self, series: str, from_date: str, to_date: str) -> Dict[str, Any]:
        """Fetch real data from Riksbank API"""
        try:
            config = API_CONFIG["Riksbanken"]
            url = config["base_url"] + config["endpoints"]["observations"].format(
                series=series, from_date=from_date, to_date=to_date
            )
            
            response = self.session.get(url, headers=config["headers"], timeout=30)
            response.raise_for_status()
            
            data = response.json()
            
            return {
                "status": "success",
                "data": data,
                "timestamp": datetime.now().isoformat(),
                "source": "Riksbanken"
            }
            
        except Exception as e:
            return {
                "status": "error",
                "error": str(e),
                "timestamp": datetime.now().isoformat(),
                "source": "Riksbanken"
            }
    
    def fetch_csn_data(self, table: str) -> Dict[str, Any]:
        """Fetch real data from CSN API"""
        try:
            config = API_CONFIG["CSN"]
            url = config["base_url"] + config["endpoints"]["studiemedel"].format(table=table)
            
            # PX-Web requires specific query format
            query_data = {
                "query": [],
                "response": {"format": "json"}
            }
            
            response = self.session.post(url, json=query_data, headers=config["headers"], timeout=30)
            response.raise_for_status()
            
            data = response.json()
            
            return {
                "status": "success",
                "data": data.get("data", []),
                "timestamp": datetime.now().isoformat(),
                "source": "CSN"
            }
            
        except Exception as e:
            return {
                "status": "error",
                "error": str(e),
                "timestamp": datetime.now().isoformat(),
                "source": "CSN"
            }

# Initialize fetcher
if 'fetcher' not in st.session_state:
    st.session_state.fetcher = RealDataFetcher()

if 'fetch_history' not in st.session_state:
    st.session_state.fetch_history = []

if 'continuous_mode' not in st.session_state:
    st.session_state.continuous_mode = False

# Header
st.title("🇸🇪 API Data Fetcher")
st.markdown("**Hämta RIKTIG data från svenska myndigheter och internationella organisationer**")

# Sidebar for API selection
st.sidebar.title("🔧 API Konfiguration")

# Continuous fetching toggle
continuous_mode = st.sidebar.toggle("🔄 Kontinuerlig datahämtning", value=st.session_state.continuous_mode)
st.session_state.continuous_mode = continuous_mode

if continuous_mode:
    fetch_interval = st.sidebar.slider("⏱️ Hämtningsintervall (sekunder)", 10, 300, 60)
    st.sidebar.info("Kontinuerlig hämtning aktiverad")

# API Selection
selected_api = st.sidebar.selectbox(
    "Välj API",
    list(API_CONFIG.keys()),
    help="Välj vilken datakälla du vill hämta RIKTIG data från"
)

# Dynamic form based on selected API
st.sidebar.subheader(f"⚙️ {selected_api} Inställningar")

def create_api_form(api_name: str):
    """Create dynamic form for selected API"""
    params = {}
    
    if api_name == "Skolverket":
        endpoint = st.sidebar.selectbox(
            "Endpoint",
            ["planned-educations", "skolenhetsregister", "syllabus"]
        )
        params['endpoint'] = endpoint
        
        if endpoint == "planned-educations":
            params['coordinateSystemType'] = st.sidebar.selectbox("Koordinatsystem", ["WGS84", "SWEREF99"])
            params['page'] = st.sidebar.number_input("Sida", 0, 100, 0)
            params['size'] = st.sidebar.number_input("Storlek", 1, 100, 20)
        
        elif endpoint == "skolenhetsregister":
            kommun = st.sidebar.text_input("Kommunkod (valfritt)", placeholder="t.ex. 1280")
            if kommun:
                params['kommunkod'] = kommun
        
        elif endpoint == "syllabus":
            params['studievagstyp'] = st.sidebar.selectbox("Studievägstyp", ["GY", "GR", "GRSÄR"])
    
    elif api_name == "SCB":
        st.sidebar.warning("⚠️ SCB har rate limit: 10 anrop/10 sek")
        params['query'] = st.sidebar.text_area(
            "SCB Query (JSON)",
            value='{"query": [], "response": {"format": "json"}}',
            help="Ange SCB query i JSON-format för RIKTIG data"
        )
    
    elif api_name == "Kolada":
        kpi_code = st.sidebar.text_input("KPI-kod (valfritt)", placeholder="t.ex. N15020")
        municipality = st.sidebar.text_input("Kommun (valfritt)", placeholder="t.ex. 1280")
        year = st.sidebar.number_input("År (valfritt)", 2000, 2024, 2023)
        
        if kpi_code:
            params['kpi'] = kpi_code
        if municipality:
            params['municipality'] = municipality
        if year:
            params['year'] = year
    
    elif api_name == "Världsbanken":
        country = st.sidebar.text_input("Land (ISO-kod)", value="se", placeholder="t.ex. se")
        indicator = st.sidebar.selectbox(
            "Indikator",
            ["NY.GDP.MKTP.CD", "SP.POP.TOTL", "SE.XPD.TOTL.GD.ZS"],
            format_func=lambda x: {
                "NY.GDP.MKTP.CD": "BNP (USD)",
                "SP.POP.TOTL": "Total befolkning", 
                "SE.XPD.TOTL.GD.ZS": "Utbildningsutgifter (% av BNP)"
            }.get(x, x)
        )
        year = st.sidebar.number_input("År", 1960, 2024, 2023)
        
        params['country'] = country
        params['indicator'] = indicator
        params['year'] = year
    
    elif api_name == "Riksbanken":
        series = st.sidebar.selectbox(
            "Dataserie",
            ["SEKEURPMI", "SEKUSDPMI", "SECOVERRPO"],
            format_func=lambda x: {
                "SEKEURPMI": "EUR/SEK kurs",
                "SEKUSDPMI": "USD/SEK kurs",
                "SECOVERRPO": "Reporänta"
            }.get(x, x)
        )
        from_date = st.sidebar.date_input("Från datum", datetime.now() - timedelta(days=30))
        to_date = st.sidebar.date_input("Till datum", datetime.now())
        
        params['series'] = series
        params['from_date'] = from_date.strftime('%Y-%m-%d')
        params['to_date'] = to_date.strftime('%Y-%m-%d')
    
    elif api_name == "CSN":
        table = st.sidebar.selectbox(
            "Tabell",
            ["SS0101B1", "SS0201A1"],
            format_func=lambda x: {
                "SS0101B1": "Studiemedel högskola",
                "SS0201A1": "Återbetalning studielån"
            }.get(x, x)
        )
        params['table'] = table
    
    return params

# Create form for selected API
form_params = create_api_form(selected_api)

# Main content area
col1, col2 = st.columns([2, 1])

with col1:
    st.subheader(f"📊 {selected_api} RIKTIG Data")
    
    # Fetch button
    if st.button("🚀 Hämta RIKTIG Data", type="primary"):
        with st.spinner(f"Hämtar RIKTIG data från {selected_api}..."):
            
            try:
                # Call real API based on selection
                if selected_api == "Skolverket":
                    endpoint = form_params.get('endpoint', 'planned-educations')
                    clean_params = {k: v for k, v in form_params.items() if k != 'endpoint'}
                    result = st.session_state.fetcher.fetch_skolverket_data(endpoint, clean_params)
                
                elif selected_api == "SCB":
                    result = st.session_state.fetcher.fetch_scb_data(form_params['query'])
                
                elif selected_api == "Kolada":
                    result = st.session_state.fetcher.fetch_kolada_data(
                        form_params.get('kpi'),
                        form_params.get('municipality'), 
                        form_params.get('year')
                    )
                
                elif selected_api == "Världsbanken":
                    result = st.session_state.fetcher.fetch_worldbank_data(
                        form_params['country'],
                        form_params['indicator'],
                        form_params['year']
                    )
                
                elif selected_api == "Riksbanken":
                    result = st.session_state.fetcher.fetch_riksbank_data(
                        form_params['series'],
                        form_params['from_date'],
                        form_params['to_date']
                    )
                
                elif selected_api == "CSN":
                    result = st.session_state.fetcher.fetch_csn_data(form_params['table'])
                
                else:
                    result = {
                        "status": "error",
                        "error": f"API {selected_api} inte implementerat än",
                        "timestamp": datetime.now().isoformat(),
                        "source": selected_api
                    }
                
                # Add to history
                st.session_state.fetch_history.append(result)
                
                # Display results
                if result["status"] == "success":
                    st.success(f"✅ RIKTIG data hämtad från {selected_api}")
                    
                    # Convert to DataFrame if possible
                    data = result["data"]
                    if isinstance(data, list) and len(data) > 0:
                        df = pd.DataFrame(data)
                        st.dataframe(df, use_container_width=True)
                        
                        # Download buttons
                        col_json, col_csv = st.columns(2)
                        with col_json:
                            st.download_button(
                                "📄 Ladda ner JSON",
                                data=json.dumps(data, indent=2, ensure_ascii=False, default=str),
                                file_name=f"{selected_api}_REAL_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json",
                                mime="application/json"
                            )
                        with col_csv:
                            csv_buffer = io.StringIO()
                            df.to_csv(csv_buffer, index=False)
                            st.download_button(
                                "📊 Ladda ner CSV",
                                data=csv_buffer.getvalue(),
                                file_name=f"{selected_api}_REAL_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",
                                mime="text/csv"
                            )
                    else:
                        st.json(data)
                else:
                    st.error(f"❌ Fel vid hämtning från {selected_api}: {result.get('error', 'Okänt fel')}")
                
            except Exception as e:
                st.error(f"❌ Kritiskt fel: {str(e)}")
                st.session_state.fetch_history.append({
                    "status": "error",
                    "error": str(e),
                    "timestamp": datetime.now().isoformat(),
                    "source": selected_api
                })

with col2:
    st.subheader("📈 Hämtningshistorik")
    
    if st.session_state.fetch_history:
        # Show last 5 fetches
        recent_fetches = st.session_state.fetch_history[-5:]
        
        for i, fetch in enumerate(reversed(recent_fetches)):
            with st.expander(f"{fetch['source']} - {fetch['timestamp'][:19]}"):
                if fetch['status'] == 'success':
                    st.success("✅ Lyckad hämtning (RIKTIG DATA)")
                    if isinstance(fetch.get('data'), list):
                        st.metric("Antal rader", len(fetch['data']))
                    elif isinstance(fetch.get('data'), dict):
                        st.metric("Dataobjekt", "1 objekt")
                else:
                    st.error("❌ Misslyckad hämtning")
                    st.text(fetch.get('error', 'Okänt fel'))
    else:
        st.info("Ingen hämtningshistorik än")
    
    # Clear history button
    if st.button("🗑️ Rensa historik"):
        st.session_state.fetch_history = []
        st.rerun()

# Continuous fetching implementation
if continuous_mode and selected_api:
    st.info(f"🔄 Kontinuerlig hämtning aktiverad för {selected_api}")
    st.info(f"⏱️ Hämtar RIKTIG data var {fetch_interval} sekunder")
    
    # Manual refresh button for continuous mode
    if st.button("🔄 Hämta nu (kontinuerlig mode)", type="secondary"):
        st.rerun()
    
    # Note about continuous fetching
    st.warning("⚠️ Kontinuerlig hämtning är aktiverad. Använd knappen ovan för manuell uppdatering av RIKTIG data.")

# Footer
st.markdown("---")
st.markdown("**🇸🇪 API Data Fetcher** - Hämtar RIKTIG data från svenska myndigheter")
st.markdown("✅ **ALLA API:ER ANVÄNDER RIKTIGA ENDPOINTS** - Inga mock-data!")
st.markdown("Stödjer: Skolverket, SCB, Kolada, Världsbanken, Riksbanken, CSN")