Update rss_processor.py

rss_processor.py

@@ -302,11 +302,18 @@ def upload_to_hf_hub():
         with open('local_rss_store.json','r') as f:
             json_data=f.read()
         f.close()
-        dataset = Dataset.from_list(json.loads(json_data)).to_parquet(f'{datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}')
+        json_list = json.loads(json_data)  # json_data is your JSON string
+        timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
+        local_filename = f'{timestamp}.parquet'
         
-        # 3. Create (if needed) and upload as Parquet
-        # This will create 'data/train-00000-of-00001.parquet' on the Hub
-        dataset.push_to_hub(DATASET_REPO_ID)
+        Dataset.from_list(json_list).to_parquet(local_filename)
+
+        api.upload_file(
+            path_or_fileobj=local_filename,
+            path_in_repo=f"data/{f"{timestamp}.parquet"}", # Recommended to keep in a 'data/' folder
+            repo_id=DATASET_REPO_ID,
+            repo_type="dataset"
+        )
 
         logger.info(f"Raw feeds folder 'local' uploaded to: {DATASET_REPO_ID}")
     except Exception as e: