Upload 2 files

update_streamlit_panel.py

@@ -0,0 +1,130 @@
+# update_streamlit_panel.py
+import sys, json, subprocess
+from pathlib import Path
+from datetime import datetime
+import streamlit as st
+
+UPDATE_CMD = [sys.executable, "update_data.py"]
+OUTPUTS   = [Path("data/known_bills_visualize.json")]
+LOCKFILE  = Path(".update.lock")
+LOGFILE   = Path("pipeline_last_run.log")
+
+def _is_lfs_pointer(path: Path) -> bool:
+    try:
+        with path.open("r", encoding="utf-8", errors="ignore") as f:
+            head = "".join([next(f) for _ in range(3)])
+        return head.startswith("version https://git-lfs.github.com/spec/v1")
+    except Exception:
+        return False
+
+def _read_tail(path: Path, max_bytes=20000) -> str:
+    if not path.exists():
+        return ""
+    data = path.read_bytes()
+    return data[-max_bytes:].decode("utf-8", errors="ignore")
+
+def _inspect_json(path: Path) -> tuple[bool, str]:
+    if not path.exists():
+        return False, f"Not found: {path}"
+    if _is_lfs_pointer(path):
+        return False, (
+            f"`{path}` looks like a Git-LFS pointer (not real data).\n"
+            "   Fix locally:\n"
+            "     git lfs install\n"
+            "     git lfs pull\n"
+            "     git lfs checkout"
+        )
+    try:
+        with path.open("r", encoding="utf-8") as f:
+            data = json.load(f)
+        count = len(data) if isinstance(data, list) else len(data.get("data", [])) if isinstance(data, dict) else 0
+        mtime = datetime.fromtimestamp(path.stat().st_mtime).strftime("%Y-%m-%d %H:%M:%S")
+        return True, f"`{path}` • rows: **{count}** • saved: **{mtime}**"
+    except Exception as e:
+        return False, f"`{path}` failed to parse as JSON: {e}"
+
+def _run_update(answer: str):
+    if LOCKFILE.exists():
+        st.error("Another update appears to be running (.update.lock exists). If stuck, delete that file and try again.")
+        return
+
+    try:
+        LOCKFILE.write_text(datetime.utcnow().isoformat())
+
+        with st.status("Running update_data.py …", expanded=True) as status:
+            st.write(f"Command: `{UPDATE_CMD}`")
+            with LOGFILE.open("w", encoding="utf-8") as lf:
+                proc = subprocess.Popen(
+                    UPDATE_CMD,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    stdin=subprocess.PIPE,
+                    text=True,
+                    bufsize=1,
+                )
+                # forward user's choice to update_data.py
+                try:
+                    proc.stdin.write(answer)
+                    proc.stdin.flush()
+                except Exception:
+                    pass
+
+                for line in proc.stdout:
+                    line = line.rstrip("\n")
+                    st.write(line)
+                    lf.write(line + "\n")
+                rc = proc.wait()
+
+            if rc == 0:
+                status.update(label="✅ Update finished", state="complete")
+                all_ok = True
+                for p in OUTPUTS:
+                    ok, msg = _inspect_json(p)
+                    (st.success if ok else st.error)(msg)
+                    all_ok &= ok
+
+                st.cache_data.clear()
+                if all_ok:
+                    st.toast("Data updated and cache cleared.")
+                    st.rerun()
+                else:
+                    st.warning("Update completed, but one or more outputs failed validation.")
+            else:
+                status.update(label="Update failed", state="error")
+                st.error(f"`update_data.py` exited with code {rc}. Log tail below:")
+                st.code(_read_tail(LOGFILE), language="bash")
+    finally:
+        try: LOCKFILE.unlink()
+        except Exception: pass
+
+def render_update_panel():
+    st.title("Admin — Data Update Panel")
+    st.caption("Hidden page (URL-only).")
+
+    c1, c2, c3 = st.columns(3)
+    run_btn     = c1.button("Run update_data.py")
+    refresh_btn = c2.button("Clear app cache")
+    tail_btn    = c3.button("Show last log tail")
+
+    if refresh_btn:
+        st.cache_data.clear()
+        st.success("Cache cleared — new data will load on next run.")
+        st.rerun()
+
+    if tail_btn:
+        st.code(_read_tail(LOGFILE) or "No previous log.", language="bash")
+
+    st.markdown("**Current local datasets**")
+    for p in OUTPUTS:
+        ok, msg = _inspect_json(p)
+        (st.success if ok else st.warning)(msg)
+
+    if run_btn:
+        choice = st.radio(
+            "Pull new data from LegiScan?",
+            ["No (use existing data)", "Yes (pull new data)"],
+            horizontal=True,
+        )
+        confirm = st.button("Confirm and Start Update", type="primary")
+        if confirm:
+            _run_update("y\n" if "Yes" in choice else "n\n")

user_management.py

@@ -0,0 +1,302 @@
+"""
+User Management Module for HuggingFace-backed authentication
+Stores user credentials in HuggingFace dataset for persistence
+"""
+
+import json
+import requests
+from huggingface_hub import HfApi
+import streamlit as st
+from pathlib import Path
+
+
+class HuggingFaceUserManager:
+    """Manage users stored in HuggingFace dataset"""
+    
+    def __init__(self):
+        """Initialize the user manager"""
+        if "huggingface" not in st.secrets:
+            raise ValueError("HuggingFace configuration missing from secrets")
+        
+        self.token = st.secrets["huggingface"]["token"]
+        self.repo_id = st.secrets["huggingface"]["dataset_repo"]
+        self.users_file = "users.json"
+        self.api = HfApi()
+    
+    def get_users_url(self):
+        """Get the URL to the users.json file"""
+        return f"https://huggingface.co/datasets/{self.repo_id}/resolve/main/{self.users_file}"
+    
+    def load_users(self, use_cache=True):
+        """
+        Load users from HuggingFace dataset
+        
+        Args:
+            use_cache (bool): Whether to use cached data
+        
+        Returns:
+            dict: User credentials in the format:
+                {
+                    'usernames': {
+                        'username': {
+                            'email': 'email@example.com',
+                            'name': 'Full Name',
+                            'password': '$2b$12$hash...'
+                        }
+                    }
+                }
+        """
+        try:
+            url = self.get_users_url()
+            if not use_cache:
+                import time
+                url = f"{url}?t={int(time.time())}"
+            
+            response = requests.get(url, timeout=30)
+            response.raise_for_status()
+            
+            users_data = response.json()
+            
+            if 'usernames' not in users_data:
+                users_data = {'usernames': {}}
+            
+            return users_data
+            
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 404:
+                print("users.json not found on HuggingFace, creating new structure")
+                return {'usernames': {}}
+            else:
+                raise
+        except Exception as e:
+            print(f"Error loading users from HuggingFace: {e}")
+            return {'usernames': {}}
+    
+    def save_users(self, users_data):
+        """
+        Save users to HuggingFace dataset
+        
+        Args:
+            users_data (dict): User credentials dictionary
+            
+        Returns:
+            str: Commit URL
+        """
+        try:
+            temp_file = Path("temp_users.json")
+            with temp_file.open('w', encoding='utf-8') as f:
+                json.dump(users_data, f, indent=2, ensure_ascii=False)
+            
+            # Upload to HuggingFace
+            commit_info = self.api.upload_file(
+                path_or_fileobj=str(temp_file),
+                path_in_repo=self.users_file,
+                repo_id=self.repo_id,
+                repo_type="dataset",
+                token=self.token,
+                commit_message="Update user credentials"
+            )
+            
+            temp_file.unlink()
+            
+            if isinstance(commit_info, str):
+                return commit_info
+            elif hasattr(commit_info, 'commit_url'):
+                return commit_info.commit_url
+            else:
+                return f"https://huggingface.co/datasets/{self.repo_id}"
+            
+        except Exception as e:
+            if temp_file.exists():
+                temp_file.unlink()
+            raise Exception(f"Failed to save users to HuggingFace: {str(e)}")
+    
+    def add_user(self, username, email, name, hashed_password):
+        """
+        Add a new user
+        
+        Args:
+            username (str): Username
+            email (str): Email address
+            name (str): Full name
+            hashed_password (str): Bcrypt hashed password
+            
+        Returns:
+            tuple: (success: bool, message: str, commit_url: str|None)
+        """
+        try:
+            # Load current users
+            users_data = self.load_users()
+            
+            # Check if username already exists
+            if username in users_data['usernames']:
+                return False, f"Username '{username}' already exists", None
+            
+            # Add new user
+            users_data['usernames'][username] = {
+                'email': email,
+                'name': name,
+                'password': hashed_password
+            }
+            
+            # Save to HuggingFace
+            commit_url = self.save_users(users_data)
+            
+            return True, f"User '{username}' added successfully", commit_url
+            
+        except Exception as e:
+            return False, f"Error adding user: {str(e)}", None
+    
+    def remove_user(self, username):
+        """
+        Remove a user
+        
+        Args:
+            username (str): Username to remove
+            
+        Returns:
+            tuple: (success: bool, message: str, commit_url: str|None)
+        """
+        try:
+            users_data = self.load_users()
+            
+            if username not in users_data['usernames']:
+                return False, f"Username '{username}' not found", None
+            
+            # Remove user
+            del users_data['usernames'][username]
+            
+            # Save to HuggingFace
+            commit_url = self.save_users(users_data)
+            
+            return True, f"User '{username}' removed successfully", commit_url
+            
+        except Exception as e:
+            return False, f"Error removing user: {str(e)}", None
+    
+    def update_user_password(self, username, new_hashed_password):
+        """
+        Update a user's password
+        
+        Args:
+            username (str): Username
+            new_hashed_password (str): New bcrypt hashed password
+            
+        Returns:
+            tuple: (success: bool, message: str, commit_url: str|None)
+        """
+        try:
+            # Load current users
+            users_data = self.load_users()
+            
+            # Check if user exists
+            if username not in users_data['usernames']:
+                return False, f"Username '{username}' not found", None
+            
+            # Update password
+            users_data['usernames'][username]['password'] = new_hashed_password
+            
+            # Save to HuggingFace
+            commit_url = self.save_users(users_data)
+            
+            return True, f"Password updated for '{username}'", commit_url
+            
+        except Exception as e:
+            return False, f"Error updating password: {str(e)}", None
+    
+    def update_user(self, username, new_email=None, new_name=None, new_password=None):
+        """
+        Update a user's details
+        
+        Args:
+            username (str): Username
+            new_email (str, optional): New email address
+            new_name (str, optional): New full name
+            new_password (str, optional): New bcrypt hashed password
+            
+        Returns:
+            tuple: (success: bool, message: str, commit_url: str|None)
+        """
+        try:
+            # Load current users
+            users_data = self.load_users()
+            
+            # Check if user exists
+            if username not in users_data['usernames']:
+                return False, f"Username '{username}' not found", None
+            
+            # Update fields if provided
+            if new_email is not None:
+                users_data['usernames'][username]['email'] = new_email
+            if new_name is not None:
+                users_data['usernames'][username]['name'] = new_name
+            if new_password is not None:
+                users_data['usernames'][username]['password'] = new_password
+            
+            # Save to HuggingFace
+            commit_url = self.save_users(users_data)
+            
+            return True, f"User '{username}' updated successfully", commit_url
+            
+        except Exception as e:
+            return False, f"Error updating user: {str(e)}", None
+    
+    def get_config_for_authenticator(self):
+        """
+        Get user data formatted for streamlit-authenticator
+        
+        Returns:
+            dict: Configuration dictionary for Authenticate()
+        """
+        users_data = self.load_users()
+        
+        # Get cookie config from secrets
+        cookie_config = {
+            'name': st.secrets.get('auth', {}).get('cookie', {}).get('name', 'auth_cookie'),
+            'key': st.secrets.get('auth', {}).get('cookie', {}).get('key', 'random_key'),
+            'expiry_days': int(st.secrets.get('auth', {}).get('cookie', {}).get('expiry_days', 30))
+        }
+        
+        return {
+            'credentials': users_data,
+            'cookie': cookie_config,
+            'preauthorized': {'emails': []}
+        }
+
+
+@st.cache_data(ttl=300)  
+def load_user_config():
+    """
+    Load user configuration from HuggingFace or fallback to secrets.toml
+    
+    Returns:
+        tuple: (config: dict, using_hf: bool)
+    """
+    try:
+        if "huggingface" in st.secrets:
+            manager = HuggingFaceUserManager()
+            config = manager.get_config_for_authenticator()
+            return config, True
+    except Exception as e:
+        print(f"Could not load from HuggingFace: {e}")
+    
+    if "auth" in st.secrets:
+        def _secrets_to_dict(obj):
+            if hasattr(obj, 'to_dict'):
+                return obj.to_dict()
+            elif isinstance(obj, dict):
+                return {k: _secrets_to_dict(v) for k, v in obj.items()}
+            elif isinstance(obj, (list, tuple)):
+                return [_secrets_to_dict(item) for item in obj]
+            else:
+                return obj
+        
+        auth_secrets = _secrets_to_dict(st.secrets["auth"])
+        config = {
+            'credentials': auth_secrets.get('credentials', {}),
+            'cookie': auth_secrets.get('cookie', {}),
+            'preauthorized': auth_secrets.get('preauthorized', {'emails': []})
+        }
+        return config, False
+    
+    return None, False