Update rss_processor.py

rss_processor.py

@@ -12,9 +12,7 @@ import json
 import re
 import requests
 import pandas as pd
-from datasets import Dataset
-
-
+from datasets import Dataset, load_dataset, concatenate_datasets
 
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -296,29 +294,48 @@ def upload_to_hf_hub():
         except Exception as e:
             logger.error(f"Error uploading raw feeds to Hugging Face Hub: {e}", exc_info=True)
 
+
+
     try:
-        logger.info(f"Uploading raw RSS feeds from 'local' to {DATASET_REPO_ID}...")
+        logger.info(f"Processing RSS feeds for {DATASET_REPO_ID}...")
         
-        # 2. Convert list to a Hugging Face Dataset
-        with open('local_rss_store.json','r') as f:
-            json_data=f.read()
-        f.close()
-        json_list = json.loads(json_data)  # json_data is your JSON string
-        timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
-        local_filename = f'{timestamp}.parquet'
+        # 1. Load Local JSON
+        with open('local_rss_store.json', 'r') as f:
+            json_list = json.load(f)
         
-        Dataset.from_list(json_list).to_parquet(local_filename)
+        if not json_list:
+            logger.info("No local RSS data to upload.")
+            # return # Optional: Exit if empty
+        else:
+            # Create a HF Dataset object from the new local data
+            new_dataset = Dataset.from_list(json_list)
+
+            # 2. Try to Load Existing Dataset from the Hub
+            try:
+                # We load the existing dataset to append to it
+                existing_dataset = load_dataset(DATASET_REPO_ID, split="train")
+                logger.info(f"Found existing dataset with {len(existing_dataset)} rows.")
+                
+                # OPTIONAL: Align features (columns) if RSS structure changes
+                # new_dataset = new_dataset.cast(existing_dataset.features)
+                
+                # 3. Concatenate (Append)
+                final_dataset = concatenate_datasets([existing_dataset, new_dataset])
+                logger.info(f"Appending {len(new_dataset)} new rows. Total size: {len(final_dataset)}")
+            
+            except Exception as e:
+                # If dataset doesn't exist yet, start fresh
+                logger.info(f"No existing dataset found (or error loading). Creating new. Details: {e}")
+                final_dataset = new_dataset
+
+            # 4. Push the Unified Dataset back to Hub
+            # This updates the main parquet file(s) cleanly
+            final_dataset.push_to_hub(DATASET_REPO_ID)
 
-        hf_api.upload_file(
-            path_or_fileobj=local_filename,
-            path_in_repo=f"data/{timestamp}.parquet", # Recommended to keep in a 'data/' folder
-            repo_id=DATASET_REPO_ID,
-            repo_type="dataset"
-        )
+            logger.info(f"Successfully pushed updated dataset to {DATASET_REPO_ID}")
 
-        logger.info(f"Raw feeds folder 'local' uploaded to: {DATASET_REPO_ID}")
     except Exception as e:
-        logger.error(f"Error uploading raw feeds to Hugging Face Hub: {e}", exc_info=True)
+        logger.error(f"Error appending RSS feeds to Hugging Face Hub: {e}", exc_info=True)