Update app.py

app.py

@@ -1,166 +1,92 @@
-import gradio as gr
-import os
-import pandas as pd
-from huggingface_hub import HfFileSystem
-import tempfile
-
-# --- CONFIGURATION ---
-HF_BUCKET = "hf://buckets/Krixter/storage"
-
-def get_fs():
-    """Initialize the Hugging Face File System. Uses a token if available, otherwise runs anonymously."""
-    token = os.environ.get("HF_TOKEN")
-    return HfFileSystem(token=token)
-
-def format_size(size_bytes):
-    """Convert bytes to a human-readable format."""
-    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
-        if size_bytes < 1024.0:
-            return f"{size_bytes:.2f} {unit}"
-        size_bytes /= 1024.0
-    return f"{size_bytes:.2f} PB"
-
-def list_files(sort_by="Date (Newest)"):
-    """Fetch files from the bucket, extract accurate metadata, and sort."""
-    try:
-        fs = get_fs()
-        files = fs.ls(HF_BUCKET, detail=True)
-        
-        data = []
-        for f in files:
-            if f.get("type") == "directory":
-                continue
-                
-            name = os.path.basename(f["name"])
-            raw_size = f.get("size", 0)
-            size_str = format_size(raw_size)
-            
-            # --- AGGRESSIVE DATE EXTRACTION ---
-            raw_date = None
-            
-            # Prioritizing CREATED date keys as requested, falling back to modified keys
-            date_keys = [
-                "created", "created_at", "CreatedAt", "CreationDate", "ctime", "date_created",
-                "last_modified", "LastModified", "updated", "updatedAt", "client_modified"
-            ]
-            
-            # 1. Check the primary file metadata dictionary
-            for k in date_keys:
-                if k in f and f[k] is not None:
-                    raw_date = f[k]
-                    break
-            
-            # 2. If still missing, ping info() directly for deeper hidden keys
-            if raw_date is None:
-                try:
-                    info = fs.info(f["name"])
-                    for k in date_keys:
-                        if k in info and info[k] is not None:
-                            raw_date = info[k]
-                            break
-                except Exception:
-                    pass
-            
-            data.append({
-                "File Name": name,
-                "Size": size_str,
-                "_raw_date": raw_date,
-                "_raw_size": raw_size
-            })
-            
-        df = pd.DataFrame(data)
-        if not df.empty:
-            # Safely parse dates. utc=True prevents timezone conflicts from turning valid dates into "Unknown" (NaT)
-            dt_col = pd.to_datetime(df["_raw_date"], errors='coerce', utc=True).dt.tz_localize(None)
-            df["_raw_date"] = dt_col
-            
-            # Format display column, filling NaT back to "Unknown"
-            df["Date Created"] = dt_col.dt.strftime('%Y-%m-%d %H:%M:%S').fillna("Unknown")
-            
-            # --- EXPLICIT SORTING ---
-            if sort_by == "Name (A-Z)":
-                df = df.sort_values(by="File Name", ascending=True)
-            elif sort_by == "Name (Z-A)":
-                df = df.sort_values(by="File Name", ascending=False)
-            elif sort_by == "Date (Newest)":
-                df = df.sort_values(by="_raw_date", ascending=False, na_position='last')
-            elif sort_by == "Date (Oldest)":
-                df = df.sort_values(by="_raw_date", ascending=True, na_position='last')
-            elif sort_by == "Size (Largest)":
-                df = df.sort_values(by="_raw_size", ascending=False)
-            elif sort_by == "Size (Smallest)":
-                df = df.sort_values(by="_raw_size", ascending=True)
-                
-            # Clean up raw columns and enforce column order
-            df = df.drop(columns=["_raw_date", "_raw_size"])
-            df = df[["File Name", "Date Created", "Size"]]
-            
-        file_names = df["File Name"].tolist() if not df.empty else []
-        return df, gr.update(choices=file_names)
-        
-    except Exception as e:
-        print(f"Error fetching files: {e}")
-        return pd.DataFrame(columns=["File Name", "Date Created", "Size"]), gr.update(choices=[])
-
-def download_file(filename):
-    """Download the selected file from the bucket to a temporary local file."""
-    if not filename:
-        return None
-    
-    try:
-        fs = get_fs()
-        src_path = f"{HF_BUCKET}/{filename}"
-        
-        temp_dir = tempfile.mkdtemp()
-        dest_path = os.path.join(temp_dir, filename)
-        
-        print(f"📥 Downloading {filename}...")
-        fs.get(src_path, dest_path)
-        print("✅ Download complete!")
-        
-        return dest_path
-    except Exception as e:
-        print(f"⚠️ Error downloading {filename}: {e}")
-        return None
-
-# --- UI LAYOUT ---
-with gr.Blocks() as app:
-    gr.Markdown("# 🗄️ AudioNovel Bucket Explorer")
-    gr.Markdown("View, sort, and download your processed audio files. Use the dropdown to sort your files perfectly.")
-    
-    with gr.Row():
-        with gr.Column(scale=2):
-            refresh_btn = gr.Button("🔄 Refresh Bucket contents", variant="secondary")
-        with gr.Column(scale=1):
-            sort_dropdown = gr.Dropdown(
-                label="Sort Files By",
-                choices=["Name (A-Z)", "Name (Z-A)", "Date (Newest)", "Date (Oldest)", "Size (Largest)", "Size (Smallest)"],
-                value="Date (Newest)",
-                interactive=True
-            )
-            
-    # Interactive table (Headers updated)
-    file_table = gr.Dataframe(
-        headers=["File Name", "Date Created", "Size"],
-        interactive=False,
-        label="Bucket Contents"
-    )
-    
-    gr.Markdown("---")
-    gr.Markdown("### 📥 Download a File")
-    
-    with gr.Row():
-        with gr.Column(scale=2):
-            file_dropdown = gr.Dropdown(label="Select File to Download", choices=[])
-        with gr.Column(scale=1):
-            download_btn = gr.Button("🚀 Prepare Download", variant="primary")
-            
-    file_output = gr.File(label="Ready for Download")
-    
-    # --- EVENT TRIGGERS ---
-    refresh_btn.click(fn=list_files, inputs=[sort_dropdown], outputs=[file_table, file_dropdown])
-    sort_dropdown.change(fn=list_files, inputs=[sort_dropdown], outputs=[file_table, file_dropdown])
-    app.load(fn=list_files, inputs=[sort_dropdown], outputs=[file_table, file_dropdown])
+"""
+Webnovel.com NTR keyword scraper
+Searches Webnovel's public search API for titles/descriptions
+containing "netorase", "netorare", or "ntr" and saves results to CSV.
+
+Run with: python webnovel_ntr_scraper.py
+Requires: pip install requests
+"""
+
+import requests
+import csv
+import time
+import json
+
+KEYWORDS = ["netorase", "netorare", "ntr"]
+OUTPUT_FILE = "webnovel_ntr_results.csv"
+
+# Webnovel's internal search API (subject to change)
+SEARCH_URL = "https://www.webnovel.com/go/pcm/search/result"
+
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+                  "(KHTML, like Gecko) Chrome/124.0 Safari/537.36",
+    "Accept": "application/json, text/plain, */*",
+}
+
+def search_keyword(keyword, page_size=20, max_pages=20, delay=1.5):
+    """Query Webnovel's search endpoint for a keyword, paginating through results."""
+    results = []
+    for page in range(1, max_pages + 1):
+        params = {
+            "keywords": keyword,
+            "pageIndex": page,
+            "pageSize": page_size,
+        }
+        try:
+            resp = requests.get(SEARCH_URL, params=params, headers=HEADERS, timeout=10)
+            resp.raise_for_status()
+            data = resp.json()
+        except (requests.RequestException, json.JSONDecodeError) as e:
+            print(f"  [!] Error on page {page} for '{keyword}': {e}")
+            break
+
+        # Adjust this path based on actual API response structure
+        items = (
+            data.get("data", {}).get("items")
+            or data.get("data", {}).get("bookItems")
+            or []
+        )
+
+        if not items:
+            break
+
+        for item in items:
+            title = item.get("bookName") or item.get("title", "")
+            desc = item.get("description") or item.get("bookDesc", "")
+            book_id = item.get("bookId", "")
+            url = f"https://www.webnovel.com/book/{book_id}" if book_id else ""
+            results.append({"title": title, "description": desc, "url": url})
+
+        print(f"  Keyword '{keyword}' page {page}: {len(items)} items")
+        time.sleep(delay)
+
+    return results
+
+def matches_ntr(title, description, keywords):
+    text = f"{title} {description}".lower()
+    return any(k.lower() in text for k in keywords)
+
+def main():
+    all_matches = {}
+
+    for kw in KEYWORDS:
+        print(f"Searching for keyword: {kw}")
+        items = search_keyword(kw)
+        for item in items:
+            if matches_ntr(item["title"], item["description"], KEYWORDS):
+                key = item["url"] or item["title"]
+                all_matches[key] = item
+
+    print(f"\nFound {len(all_matches)} unique matching novels.")
+
+    with open(OUTPUT_FILE, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=["title", "description", "url"])
+        writer.writeheader()
+        for item in all_matches.values():
+            writer.writerow(item)
+
+    print(f"Saved results to {OUTPUT_FILE}")
 
 if __name__ == "__main__":
-    app.launch(theme=gr.themes.Soft(primary_hue="indigo"))
+    main()