praw.py

# import gradio as gr
import numpy as np
import pandas as pd
import praw
from huggingface_hub import HfApi, HfFolder
import time
import os
from datetime import datetime
# from tqdm import tqdm

HfFolder.save_token(os.getenv("HF_TOKEN"))

try:
# def initialize_reddit():


    reddit = praw.Reddit(client_id= os.getenv("PRAW_CLIENT_ID"),
                         client_secret= os.getenv("PRAW_CLIENT_SECRET"),
                         user_agent= os.getenv("RPAW_AGENT"),
                         check_for_async=False
                          )
    
except praw.exceptions.PRAWException as e:
    print(f"PRAW Exception: {str(e)}")
    # return None
except Exception as e:
    print(f"An error occurred: {str(e)}")
    # return None

def scrape_reddit(subreddit_name = None, keywords = None, limit = 1000):

    posts_data = []

    if subreddit_name:
        subreddit = reddit.subreddit(subreddit_name)
        if keywords:
            posts = subreddit.search(keywords, limit=limit)
        else:
            posts = subreddit.hot(limit=limit)
    else:
        posts = reddit.subreddit("all").search(keywords, limit=limit)
    # print(posts)
    for post in posts:
        # print(post.title)
        try:
            post_data = {
                "title": post.title,
                "score": post.score,
                "id": post.id,
                "url": post.url,
                "num_comments": post.num_comments,
                "created": datetime.fromtimestamp(post.created),
                "body": post.selftext,
                "subreddit": post.subreddit.display_name
            }
            posts_data.append(post_data)

            # Add a small delay to avoid hitting rate limits
            time.sleep(0.1)

        except praw.exceptions.PRAWException as e:
            print(f"Error processing post {post.id}: {str(e)}")
            continue

    df = pd.DataFrame(posts_data)
    df['content'] = df['title'] + '\n' + df['body']
    return df



def get_comments(reddit, post_id, limit=100):
    """
    Get top comments from a specific post.

    Args:
        reddit: Reddit instance
        post_id (str): ID of the post to get comments from
        limit (int): Maximum number of comments to retrieve (default 100)

    Returns:
        pd.DataFrame: DataFrame containing top comments data
    """
    try:
        submission = reddit.submission(id=post_id)
        comments_data = []

        # Replace MoreComments objects with actual comments, limited to save time
        submission.comments.replace_more(limit=0)  # Ignore "More Comments" expansions

        # Get all top-level comments
        all_comments = submission.comments.list()


        # Sort comments by score and take top ones
        sorted_comments = sorted(all_comments, key=lambda x: x.score, reverse=True)[:limit]

        for comment in sorted_comments:

            try:
                comment_data = {
                    'comment_id': comment.id,
                    'post_id': post_id,
                    'post_title': submission.title,
                    # 'author': str(comment.author) if comment.author else '[deleted]',
                    'body': comment.body,
                    'score': comment.score,
                    'created_utc': datetime.fromtimestamp(comment.created_utc)
                    # 'parent_id': comment.parent_id,
                    # 'is_submitter': comment.is_submitter
                }
                comments_data.append(comment_data)

            except Exception as e:
                print(f"Error processing comment {comment.id}: {str(e)}")
                continue
        print(comments_data)

        # Create DataFrame
        df = pd.DataFrame(comments_data)

        # Sort by score (highest first)
        if not df.empty:
            print("sort comments by score")
            df = df.sort_values('score', ascending=False)

        return df

    except praw.exceptions.PRAWException as e:
        print(f"PRAW Exception while getting comments: {str(e)}")
        return pd.DataFrame()
    except Exception as e:
        print(f"Error getting comments: {str(e)}")
        return pd.DataFrame()



def get_comments_and_upload(df, dataset_repo_id):
    # Initialize the Hugging Face API
    api = HfApi()

    existing_files = api.list_repo_files(repo_id=dataset_repo_id, repo_type="dataset")

    # Iterate over each submission in the DataFrame
    for index, row in df.iterrows():
        csv_file_path = f"comments_{row['id']}.csv"
        repo_csv_path = f"comments/{csv_file_path}"

        # Check if this file already exists in the Hugging Face dataset
        # if repo_csv_path in existing_files:
        #     print(f"{csv_file_path} already exists in the dataset. Skipping upload.")
        #     continue
        # Fetch comments for the current submission
        comments_df = get_comments(reddit, row['id'])

        if len(comments_df) == 0:
            print(f"No comments found for {row['id']}")
            # continue
        # Define a unique CSV filename for each submission based on its ID
        csv_file_path = f"comments_{row['id']}.csv"

        # Save the comments DataFrame as a CSV file
        comments_df.to_csv(csv_file_path, index=False)

        # Upload the CSV file to the Hugging Face dataset repository
        api.upload_file(
            path_or_fileobj=csv_file_path,
            path_in_repo=f"comments/{csv_file_path}",  # Save in a 'comments' folder in the dataset repo
            repo_id=dataset_repo_id,
            repo_type="dataset"
        )

        print(f"Uploaded {csv_file_path} to Hugging Face.")

        # Optionally, delete the local CSV file to save space
        os.remove(csv_file_path)

    print("All comments CSV files uploaded successfully!")


    
def main():
    # Example usage

    try:
        # Search for 2016 election posts
        df = scrape_reddit(keywords="election")

        if df is not None and not df.empty:
            print(f"Successfully scraped {len(df)} posts")
            # Save to CSV
            # df.to_csv("reddit_2016_election_posts.csv", index=False)
            df['created'] = pd.to_datetime(df['created'], unit='s')
            df = df.sort_values(by='created', ascending=True)
            df_24 = df[df['created'] > '2024-01-01'].reset_index(drop=True)
            # df_16 = df_16[df_16['created'] > '2015-12-31'].reset_index(drop=True)
            

            dataset_repo_id = "Vera-ZWY/reddite2024elections_submissions"
            # reate database if it's not exsit
            api = HfApi()
            try:
                api.dataset_info(dataset_repo_id)
                # dataset_exists = True
                print(f"Dataset {dataset_repo_id} already exists.")

            except Exception:
            # dataset_exists = False
                print(f"Dataset {dataset_repo_id} will be created.")
                # If the dataset doesn't exist, create it and then upload the CSV file
                # api.create_repo(repo_id=dataset_repo_id, repo_type="dataset")
                
            today_date = datetime.now().strftime('%Y%m%d')
            filename = f"df_24_{today_date}.csv"

            df_24.to_csv(filename, index=False)
            # csv_file_path = filename

            api.upload_file(
                path_or_fileobj= filename,
                path_in_repo=f"submissions/{filename}",
                repo_id=dataset_repo_id,
                repo_type="dataset"
            )

            get_comments_and_upload(df_24, dataset_repo_id)

        else:
            print("No data was retrieved")


    except Exception as e:
        print(f"Error in main: {str(e)}")
        

if __name__ == '__main__':
    main()