Oracle/deepfundingoracle.py

"""
DeepFunding Oracle:
This script dynamically loads dependency data and for each repository URL:
  • Fetches GitHub features (stars, forks, watchers, open issues, pull requests, activity) using the GitHub API.
  • Uses the LLama model to analyze parent-child behavior (based on the fetched features and parent info)
    and returns a base weight (0-1) for the repository.
  • Trains a RandomForest regressor on these features (with the base weight as the target) to predict a final weight.
The output submission CSV has three columns: repo, parent, and final_weight.
"""
import base64
from io import StringIO
import os
import warnings
import csv
import re
import requests
import numpy as np
import pandas as pd
import time
import threading
import logging
import concurrent.futures
from concurrent.futures import ThreadPoolExecutor
import signal

from sklearn.pipeline import Pipeline
from tqdm import tqdm
import sys
import re
import json
import time
import json
import time
import logging
import sys
import warnings
import concurrent.futures
from concurrent.futures import ThreadPoolExecutor
import numpy as np
import pandas as pd
import requests
from tqdm import tqdm
from scipy.special import log1p, expm1
from sklearn.model_selection import RandomizedSearchCV, GroupKFold
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV,KFold
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import StandardScaler
from scipy.special import log1p, expm1
from sklearn.preprocessing import RobustScaler
from sklearn.metrics import mean_squared_error
from xgboost import XGBRegressor
from scipy.special import log1p, expm1
from Oracle.SmolLM import SmolLM
import os
warnings.filterwarnings("ignore")

# Configure logging to file and console
logging.basicConfig(
    handlers=[
        logging.FileHandler("deepfundingoracle.log", mode='w'),
        logging.StreamHandler(sys.stdout)
    ],
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s")

# Add these functions to make the pipeline importable by app.py


def prepare_dataset(file_path):
    """
    Wrapper function that prepares the dataset by:
    1. Loading the CSV
    2. Fetching GitHub features
    3. Adding derived features
    4. Cleaning data
    5. Generating base weights using LLM

    Args:
        file_path: Path to the input CSV file

    Returns:
        DataFrame with all features and base_weight prepared
    """
    logging.info(f"Preparing dataset from {file_path}")

    # Load data
    if isinstance(file_path, str):
        df = pd.read_csv(file_path)
    else:
        # Handle file object (from Gradio)
        df = pd.read_csv(file_path)

    # Check required columns
    if not {"repo", "parent"}.issubset(df.columns):
        raise ValueError("Input CSV must contain 'repo' and 'parent' columns.")

    # Run the pipeline steps
    df = fetch_github_features(df)
    df = add_derived_features(df)
    df = clean_data(df)
    df = generate_all_base_weights(df)

    return df


def run_full_pipeline(input_file, output_file="submission_enhanced.csv"):
    """
    Runs the complete DeepFunding Oracle pipeline.

    Args:
        input_file: Path to input CSV file
        output_file: Path for output CSV file

    Returns:
        The processed DataFrame with final_weight column
    """
    logging.info("--- Starting DeepFunding Oracle Pipeline ---")

    # Prepare dataset
    df = prepare_dataset(input_file)

    # Train model and predict weights
    df = train_predict_weight(df)

    # Normalize weights
    df = normalize_and_clip_weights(df)

    # Save results
    create_submission_csv(df, output_file)

    logging.info("--- Pipeline Completed Successfully ---")

    return df

##############################
#  GitHub API helper: Fetch repository metrics
##############################
def fetch_repo_metrics(repo_url):
    """
    Fetches GitHub metrics, handling API pagination to get accurate
    contributor and pull request counts.
    """
    default_metrics = {
                        "stars": 0,
                        "forks": 0,
                        "watchers": 0,
                        "open_issues": 0,
                        "pulls": 0,
                        "activity": pd.NaT,
                        "created_at":pd.NaT,
                        "contributors": 0
                    }
    try:
        m = re.search(r"github.com/([^/]+)/([^/]+)", repo_url)
        if not m:
            logging.warning(f"Malformed GitHub URL: {repo_url}")
            return default_metrics
        owner, repo_name = m.group(1), m.group(2)
        api_url = f"https://api.github.com/repos/{owner}/{repo_name}"
        headers = {}
        token = os.environ.get("GITHUB_API_TOKEN")
        if token:
            headers["Authorization"] = f"token {token}"

        r = requests.get(api_url, headers=headers, timeout=15)
        r.raise_for_status()
        data = r.json()

        def get_count_from_pagination(url, headers):
            try:
                resp = requests.get(f"{url}?per_page=1", headers=headers, timeout=10)
                if resp.status_code == 200 and 'Link' in resp.headers:
                    match = re.search(r'page=(\d+)>; rel="last"', resp.headers['Link'])
                    if match:
                        return int(match.group(1))
                return len(resp.json()) if resp.status_code == 200 else 0
            except requests.exceptions.RequestException:
                return 0

        return {
            "stars": data.get("stargazers_count", 0),
            "forks": data.get("forks_count", 0),
            "watchers": data.get("subscribers_count", 0),  # subscribers_count is a better 'watch' metric
            "open_issues": data.get("open_issues_count", 0),
            "activity": pd.to_datetime(data.get("updated_at")),
            "created_at": pd.to_datetime(data.get("created_at")),
            "contributors":  get_count_from_pagination(data['contributors_url'], headers),
            "pulls": get_count_from_pagination(data['pulls_url'].replace('{/number}', ''), headers)
        }
    except requests.exceptions.RequestException as e:
        logging.error(f"Failed to fetch data for {repo_url}: {e}")
        return default_metrics

def fetch_github_features(df):
    """Concurrently fetches GitHub features for all repositories in the DataFrame."""
    logging.info("Fetching GitHub features for repositories...")
    metrics_data = []
    with ThreadPoolExecutor(max_workers=20) as executor:
        future_to_url = {executor.submit(fetch_repo_metrics, url): url for url in df['repo']}
        for future in tqdm(concurrent.futures.as_completed(future_to_url), total=len(df), desc="Fetching GitHub Metrics"):
            metrics_data.append(future.result())
    return pd.concat([df.reset_index(drop=True), pd.DataFrame(metrics_data)], axis=1)


def add_derived_features(df):
    """
    RATIONALE (Recommendation 2): Adds derived temporal and interaction features like 'days_since_update'
    and 'stars_per_contributor' to give the model more powerful signals to learn from.
    """
    logging.info("Engineering derived features...")

    # Handle timestamp
    df['activity'] = pd.to_datetime(df['activity'], errors='coerce',utc=True)
    df['created_at'] = pd.to_datetime(df['created_at'], errors='coerce', utc=True)

    # Temporal features
    now = pd.Timestamp.now(tz='UTC')
    df['days_since_update'] = (now - df['activity']).dt.days
    df['repo_age_days'] = (now - df['created_at']).dt.days

    #Interactions and Ratio Features
    df['stars_per_contributor'] = df['stars'] / df['contributors'].clip(lower=1)
    df['forks_per_star'] = df['forks'] / df['stars'].clip(lower=1)
    df ['pulls_per_contributor'] = df['pulls'] / df['contributors'].clip(lower=1)
    df['stars_per_day'] = df['stars']/df['repo_age_days'].clip(lower=1)

    #Dependency features
    df['dependencies_count']= df.groupby('parent')['repo'].transform('count')

    numeric_cols = df.select_dtypes(include=np.number).columns
    df[numeric_cols] = df[numeric_cols].fillna(df[numeric_cols].median())
    return df

##############################
#  Feature Extraction
##############################


def assign_base_weight_per_row(row, oracle):
    """
    RATIONALE: Asks the LLM to score each repo individually on a 0-1 scale based on its specific stats.
    This creates a much more accurate and nuanced target variable `base_weight` for the XGBoost model.
    """
    stats = (f"Stars: {int(row.get('stars', 0))}, "
             f"Contributors: {int(row.get('contributors', 0))}, "
             f"Pull Requests: {int(row.get('pulls', 0))}, "
             f"Days Since Last Update: {int(row.get('days_since_update', 0))}, "
             f"Repo Age (Days): {int(row.get('repo_age_days', 0))}")

    # <<< UPDATED: The prompt now asks for a score on a 0.0 to 1.0 scale. >>>
    prompt = (
        "You are a venture capitalist analyzing open-source projects for the Ethereum ecosystem. "
        f"A project named '{row['repo'].split('/')[-1]}' is a dependency of '{row['parent'].split('/')[-1]}'. "
        "Here are its key metrics: "
        f"[{stats}]. "
        "Based on these metrics, evaluate its standalone importance, community health, and development velocity. "
        "On a scale of 0.0 (minor, easily replaceable utility) to 1.0 (critical, foundational dependency), how would you score this project's value to its parent? "
        "Provide ONLY the numeric score in your answer. Example: 0.8"
    )

    try:
        response = oracle.predict(prompt, max_new_tokens=10)
        match = re.search(r"(\d+(\.\d+)?)", response)
        if match:
            return max(0.0, min(1.0, float(match.group(1))))
        logging.warning(f"Could not parse score from LLM for {row['repo']}. Defaulting.")
        return 0.4
    except Exception as e:
        logging.error(f"LLM prediction failed for {row['repo']}: {e}. Defaulting.")
        return 0.4

def generate_all_base_weights(df):
    """Applies the per-row LLM evaluation to the entire dataframe."""
    logging.info("Assigning robust base weights using per-row LLM evaluation...")
    oracle = SmolLM()
    if not oracle.available:
        logging.error("Oracle (LLM) is not available. Falling back to composite score.")
        df['base_weight'] = (
                np.log1p(df['stars']) * 0.4 +
                np.log1p(df['contributors']) * 0.4 +
                np.log1p(df['pulls']) * 0.2
        )
        return df

    df['base_weight'] = df.progress_apply(lambda row: assign_base_weight_per_row(row, oracle), axis=1)

    # This normalization step is kept as a crucial safeguard. It ensures the final `base_weight`
    # is scaled relative to its siblings, even if the LLM isn't perfectly calibrated.
    df['base_weight'] = df.groupby("parent")["base_weight"].transform(
        lambda s: (s - s.min()) / (s.max() - s.min() if s.max() > s.min() else 0.0)
    ).fillna(0.5)

    return df



def normalize_and_clip_weights(df, group_col="parent", weight_col="final_weight"):
    """Ensures final weights are non-negative and sum to 1 per group."""
    logging.info("Normalizing final weights...")
    df[weight_col] = df[weight_col].clip(lower=0)
    group_sums = df.groupby(group_col)[weight_col].transform('sum')

    # Normalize where sum > 0
    df[weight_col] = np.where(group_sums > 0, df[weight_col] / group_sums, 0)

    # Handle groups where the sum was 0 by distributing weight equally
    zero_sum_parents = df[group_sums == 0][group_col].unique()
    for parent in zero_sum_parents:
        mask = df[group_col] == parent
        count = mask.sum()
        if count > 0:
            df.loc[mask, weight_col] = 1 / count
    return df


##############################
#  Data Cleaning
##############################
def clean_data(df):
    """
    INTEGRATED: Cleans the DataFrame by imputing missing values and clipping extreme
    outliers, which helps stabilize the model.
    """
    logging.info("Cleaning data and handling outliers...")
    numeric_cols = df.select_dtypes(include=np.number).columns
    for col in numeric_cols:
        df[col].fillna(df[col].median(), inplace=True)

    # REFINED: Clip outliers using a wider percentile range (5% and 95%) which is often
    # more suitable for heavily skewed data like GitHub stats.
    for col in numeric_cols:
        if col not in ['repo_age_days', 'days_since_update']: # Don't clip age features
            q_low = df[col].quantile(0.05)
            q_high = df[col].quantile(0.95)
            df[col] = df[col].clip(q_low, q_high)
    return df

##############################
#  Model Training and Prediction
##############################
def train_predict_weight(df):
    """
    Trains an XGBoost Regressor with GroupKFold cross-validation and extensive hyperparameter tuning.
    """
    logging.info("Starting model training...")
    target_col = 'base_weight'
    if target_col not in df.columns or df[target_col].isnull().all():
        logging.error("Target column 'base_weight' is missing or all null. Aborting training.")
        df['final_weight'] = df['stars']
        return df

    drop_cols = ["repo", "parent", "activity", "created_at", target_col]
    feature_cols = [col for col in df.select_dtypes(include=np.number).columns if col not in drop_cols]

    X = df[feature_cols].copy().fillna(0)
    y = df[target_col]
    groups = df['parent']

    # RATIONALE (Recommendation 2): Log-transforming skewed input features helps the model by
    # making their distributions more normal, improving the performance of the regressor.
    for col in X.columns:
        if 'ratio' not in col and 'per' not in col and 'day' not in col:
            X[col] = np.log1p(X[col])


    pipeline = Pipeline([("scaler", RobustScaler()),
                         ("xgb", XGBRegressor(objective="reg:squarederror", n_jobs=-1, random_state=42, base_score=y.mean()))])

    param_dist = {
                'xgb__n_estimators': [100, 200, 300, 500],
                'xgb__max_depth': [3, 5, 7],
                'xgb__learning_rate': [0.01, 0.05, 0.1],
                'xgb__subsample': [0.7, 0.8, 0.9],
                'xgb__colsample_bytree': [0.7, 0.8, 0.9]
                }

    # RATIONALE (Recommendation 3): GroupKFold ensures that all repos from the same parent are in the
    # same fold. This prevents data leakage and gives a realistic measure of true performance.
    cv = GroupKFold(n_splits=5)

    # RATIONALE (Recommendation 4): Increasing n_iter explores more hyperparameter combinations,
    # increasing the chance of finding a better-performing model.
    search = RandomizedSearchCV(
        pipeline, param_distributions=param_dist, n_iter=25, cv=cv.split(X, y, groups),
        scoring="neg_root_mean_squared_error", verbose=1, n_jobs=-1, random_state=42
    )
    search.fit(X, y)

    best_model = search.best_estimator_
    logging.info(f"Best CV score (neg RMSE): {search.best_score_:.4f}")
    logging.info(f"Best parameters found: {search.best_params_}")

    raw_predictions = best_model.predict(X)

    # Apply a log transformation to raw predictions to stabilize variance before normalization.
    df['final_weight'] = np.log1p(raw_predictions - raw_predictions.min())

    return df


##############################
# CSV Output
##############################
def create_submission_csv(df, output_filename="submission.csv"):
    """Saves the final predictions to a CSV file."""
    # FIXED: Changed "weight" to "final_weight" to match the calculated column.
    submission_df = df[["repo", "parent", "final_weight"]]
    logging.info(f"Writing final results to {output_filename}...")
    submission_df.to_csv(output_filename, index=False)
    logging.info(f"Successfully created {output_filename}.")


if __name__ == "__main__":
    if 'GITHUB_API_TOKEN' not in os.environ:
        logging.warning("GITHUB_API_TOKEN environment variable not set. API rate limits will be low.")

    input_file = "input.csv"
    output_file = "submission_enhanced.csv"

    if not os.path.exists(input_file):
        logging.error(f"Input file not found: {input_file}. Please create it with 'repo' and 'parent' columns.")
        sys.exit(1)

    logging.info("--- Starting DeepFunding Oracle - Enhanced Process ---")

    main_df = pd.read_csv(input_file)
    main_df = fetch_github_features(main_df)
    main_df = add_derived_features(main_df)
    main_df = generate_all_base_weights(main_df)  # New LLM step with corrected prompt
    main_df = train_predict_weight(main_df)
    main_df = normalize_and_clip_weights(main_df)  # Final normalization step

    create_submission_csv(main_df, output_file)

    logging.info("--- Process Completed Successfully ---")