Upload futures_engine.py

src/services/futures_engine.py

@@ -3,12 +3,10 @@ import pandas as pd
 from dataclasses import dataclass
 from typing import List, Optional, Tuple
 
-# Integrated Docling for high-accuracy layout reconstruction
 try:
-    from docling.document_converter import DocumentConverter
-    DOCLING_AVAILABLE = True
-except ImportError:
-    DOCLING_AVAILABLE = False
+    import pypdf
+except Exception:
+    pypdf = None
 
 @dataclass
 class TokenData:
@@ -21,18 +19,22 @@ class TokenData:
     oiss: str = "-"
 
 class PDFParser:
-    """Handles 100% accuracy extraction of futures data from web-printed PDFs."""
+    """Handles extraction of tabular data from Coinalyze PDFs using regex."""
     
-    # Financial cleaning pattern for raw strings
-    CLEAN_VAL = re.compile(r'[\$\,\%\s]')
+    FINANCIAL_PATTERN = re.compile(
+        r'(\$?[+-]?[\d,\.]+[kKmMbB]?)\s+'             
+        r'(\$?[+-]?[\d,\.]+[kKmMbB]?)\s+'             
+        r'(?:([+\-]?[\d\.\,]+\%?|[\-\–\—]|N\/A)\s+)?' 
+        r'(?:([+\-]?[\d\.\,]+\%?|[\-\–\—]|N\/A)\s+)?' 
+        r'(\d*\.?\d+)'                                
+    )
 
-    # Keywords to filter out website UI elements repeated by Chrome Print
     IGNORE_KEYWORDS = {
-        'page', 'coinalyze', 'contract', 'filter', 'column', 'coins',
-        'mkt cap', 'vol 24h', 'vtmr', 'all contracts', 'custom metrics', 'watchlist'
+        'page', 'coinalyze', 'contract', 'filter', 'column',
+        'mkt cap', 'vol 24h', 'vtmr', 'coins', 'all contracts', 'custom metrics', 'watchlists'
     }
 
-    # --- Signal Helpers (Maintained 100% Original Logic) ---
+    # --- Signal Helpers (Moved inside to keep logic self-contained) ---
 
     @staticmethod
     def _oi_score_and_signal(oi_change: float) -> Tuple[int, str]:
@@ -53,105 +55,141 @@ class PDFParser:
 
     @classmethod
     def make_oiss(cls, oi_percent_str: str) -> str:
-        if not oi_percent_str or str(oi_percent_str).strip() in ['-', 'N/A']: return "-"
-        val = cls.CLEAN_VAL.sub("", str(oi_percent_str))
+        if not oi_percent_str: return "-"
+        val = oi_percent_str.replace("%", "").strip()
         try:
             oi_change = float(val) / 100
-            _, signal = cls._oi_score_and_signal(oi_change)
-            css_class = "oi-strong" if oi_change > 0 else "oi-weak" if oi_change < 0 else ""
+            score, signal = cls._oi_score_and_signal(oi_change)
+            
+            if oi_change > 0: css_class = "oi-strong"
+            elif oi_change < 0: css_class = "oi-weak"
+            else: css_class = ""
+
             sign = "+" if oi_change > 0 else ""
             if css_class:
-                return f'<span class="{css_class}">{sign}{oi_change*100:.1f}%</span> {signal}'
-            return f"{sign}{oi_change*100:.1f}% {signal}"
-        except: return "-"
+                return f'<span class="{css_class}">{sign}{oi_change*100:.0f}%</span> {signal}'
+            return f"{sign}{oi_change*100:.0f}% {signal}"
+        except Exception:
+            return "-"
 
     @classmethod
     def make_funding_signal(cls, funding_str: str) -> str:
-        if not funding_str or str(funding_str).strip() in ['-', 'N/A']: return "-"
+        if not funding_str or funding_str in ['-', 'N/A']: return "-"
         try:
-            val = float(cls.CLEAN_VAL.sub("", str(funding_str)))
+            val = float(funding_str.replace('%', '').strip())
             signal_word, css_class = cls._funding_score_and_signal(val)
+            
             if css_class:
-                return f'<span class="{css_class}">{val}%</span> <span style="font-size:0.8em;">{signal_word}</span>'
+                return f'<span class="{css_class}">{val}%</span> <span style="font-size:0.8em; color:#7f8c8d;">{signal_word}</span>'
             return f'{val}% {signal_word}'
-        except: return str(funding_str)
+        except Exception:
+            return funding_str
 
-    # --- Robust AI Extraction Logic ---
+    # --- Core Extraction Logic ---
 
     @classmethod
-    def extract(cls, path: str) -> pd.DataFrame:
-        print(f"   [!] AI Engine: Processing {path}")
-        if not DOCLING_AVAILABLE:
-            print("   [X] Deployment Error: 'docling' library is not installed.")
+    def extract(cls, path) -> pd.DataFrame:
+        print(f"   Parsing Futures PDF: {path.name}")
+        if pypdf is None:
+            print("   pypdf not available - PDF parsing disabled.")
             return pd.DataFrame()
-
+        data: List[TokenData] = []
         try:
-            # DocumentConverter uses layout models to "re-build" the grid Chrome destroyed
-            converter = DocumentConverter()
-            result = converter.convert(str(path))
-            all_tokens: List[TokenData] = []
-
-            for element, _ in result.document.iterate_items():
-                if hasattr(element, "data") and hasattr(element.data, "table"):
-                    # High-accuracy dataframe reconstruction
-                    df_raw = element.export_to_dataframe()
-                    all_tokens.extend(cls._process_rows(df_raw))
-
-            if not all_tokens:
-                print("   [!] Warning: No token data found in PDF.")
+            reader = pypdf.PdfReader(path)
+            for page in reader.pages:
+                raw = page.extract_text() or ""
+                lines = [ln.strip() for ln in raw.split("\n") if ln.strip()]
+                page_data = cls._parse_page_smart(lines)
+                data.extend(page_data)
+            print(f"   Extracted {len(data)} futures tokens")
+            if not data:
                 return pd.DataFrame()
-
-            final_df = pd.DataFrame([vars(t) for t in all_tokens])
-            
-            # Clean ticker duplicates (e.g., if a token repeats across page breaks)
-            final_df = final_df.drop_duplicates(subset=['ticker'], keep='first')
-            
-            print(f"   [+] Successfully extracted {len(final_df)} tokens.")
-            return final_df
-
+            df = pd.DataFrame([vars(t) for t in data])
+            df['ticker'] = df['ticker'].apply(lambda x: re.sub(r'[^A-Z0-9]', '', str(x).upper()))
+            df = df[df['ticker'].str.len() > 1]
+            print(f"   Valid futures tokens: {len(df)}")
+            return df
         except Exception as e:
-            print(f"   [X] Critical Engine Failure: {e}")
+            print(f"   PDF Error: {e}")
             return pd.DataFrame()
 
     @classmethod
-    def _process_rows(cls, df: pd.DataFrame) -> List[TokenData]:
-        tokens = []
-        for _, row in df.iterrows():
-            try:
-                # 1. Clean and validate the 'COIN' block
-                coin_cell = str(row[0]).strip()
-                if any(k in coin_cell.lower() for k in cls.IGNORE_KEYWORDS) or len(coin_cell) < 3:
-                    continue
-
-                # Chrome stacks Name and Ticker with \n. Split and filter empty parts.
-                parts = [p.strip() for p in coin_cell.split('\n') if p.strip()]
-                if not parts: continue
-                
-                name = parts[0]
-                ticker_raw = parts[1] if len(parts) > 1 else name
-                ticker = re.sub(r'[^A-Z0-9]', '', ticker_raw.upper())
-
-                # 2. Extract and Sanitize Columns based on Coinalyze Layout
-                # Mapping: 0:Coin, 1:Mkt Cap, 2:Vol 24h, 3:OI Chg, 4:PFR, 5:VTMR
-                mkt_cap = str(row[1]).strip()
-                volume = str(row[2]).strip()
-                
-                # Robust VTMR cleaning (handles Chrome artifacts and empty fields)
-                vtmr_raw = cls.CLEAN_VAL.sub("", str(row[5])) if len(row) > 5 else ""
+    def _parse_page_smart(cls, lines: List[str]) -> List[TokenData]:
+        financials = []
+        raw_text_lines = []
+        
+        for line in lines:
+            if any(k in line.lower() for k in cls.IGNORE_KEYWORDS):
+                continue
+            
+            fin_match = cls.FINANCIAL_PATTERN.search(line)
+            if fin_match:
+                groups = fin_match.groups()
+                mc = groups[0].replace('$', '').replace(',', '')
+                vol = groups[1].replace('$', '').replace(',', '')
+                oi_str = groups[2]
+                fund_str = groups[3]
+                vtmr = groups[4]
                 try:
-                    vtmr_float = float(vtmr_raw) if vtmr_raw else 0.0
-                except ValueError:
-                    vtmr_float = 0.0
-
-                tokens.append(TokenData(
-                    ticker=ticker,
-                    name=name,
-                    market_cap=mkt_cap,
-                    volume=volume,
-                    vtmr=vtmr_float,
-                    funding=cls.make_funding_signal(row[4]),
-                    oiss=cls.make_oiss(row[3])
-                ))
-            except Exception:
-                continue # Skip individual malformed rows to ensure 100% completion
-        return tokens
+                    float(vtmr)
+                    financials.append((mc, vol, vtmr, oi_str, fund_str))
+                except:
+                    raw_text_lines.append(line)
+            else:
+                if not line.isdigit() and len(line) > 1:
+                    raw_text_lines.append(line)
+        
+        token_pairs = []
+        i = 0
+        while i < len(raw_text_lines):
+            line = raw_text_lines[i]
+            clean_current = cls._clean_ticker_strict(line)
+            
+            if clean_current:
+                if i + 1 < len(raw_text_lines):
+                    next_line = raw_text_lines[i + 1]
+                    clean_next = cls._clean_ticker_strict(next_line)
+                    if clean_next:
+                        token_pairs.append((line, clean_next))
+                        i += 2
+                        continue
+            
+            if i + 1 < len(raw_text_lines):
+                name_candidate = raw_text_lines[i]
+                ticker_candidate_raw = raw_text_lines[i + 1]
+                ticker = cls._clean_ticker_strict(ticker_candidate_raw)
+                if ticker:
+                    token_pairs.append((name_candidate, ticker))
+                    i += 2
+                else:
+                    i += 1
+            else:
+                i += 1
+        
+        tokens: List[TokenData] = []
+        limit = min(len(token_pairs), len(financials))
+        
+        for k in range(limit):
+            name, ticker = token_pairs[k]
+            mc, vol, vtmr, oi_pct, fund_pct = financials[k]
+
+            oiss_val = cls.make_oiss(oi_pct) if oi_pct and oi_pct not in ['-', 'N/A'] else "-"
+            funding_val = cls.make_funding_signal(fund_pct)
+
+            tokens.append(TokenData(
+                ticker=ticker,
+                name=name,
+                market_cap=mc,
+                volume=vol,
+                vtmr=float(vtmr),
+                funding=funding_val,
+                oiss=oiss_val
+            ))
+        return tokens
+
+    @staticmethod
+    def _clean_ticker_strict(text: str) -> Optional[str]:
+        if len(text) > 15: return None
+        cleaned = re.sub(r'[^A-Z0-9]', '', text.upper())
+        if 2 <= len(cleaned) <= 12: return cleaned
+        return None