Delete src/evaluate.py

src/evaluate.py

@@ -1,143 +0,0 @@
-# evaluate.py
-import pandas as pd
-import os
-from collections import Counter
-import re
-from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score, classification_report
-
-SCRAPER_FOLDER = "drug_analysis_data_3months"  # Folder where scraper saves CSVs
-
-# -----------------------------
-# Load CSVs
-# -----------------------------
-csv_files = [f for f in os.listdir(SCRAPER_FOLDER) if f.endswith(".csv")]
-if not csv_files:
-    print("❌ No CSV files found in scraper folder!")
-    exit()
-
-dfs = [pd.read_csv(os.path.join(SCRAPER_FOLDER, f)) for f in csv_files]
-df = pd.concat(dfs, ignore_index=True)
-print(f"✅ Loaded {len(df)} rows from {len(csv_files)} CSV files.\n")
-
-# -----------------------------
-# General Stats
-# -----------------------------
-print("=== General Stats ===")
-print("Columns:", df.columns.tolist())
-print("Total rows:", len(df))
-print("Missing values per column:\n", df.isna().sum())
-print("\nDuplicate rows:", df.duplicated().sum())
-
-# Sample rows with missing data
-missing_rows = df[df.isna().any(axis=1)]
-if not missing_rows.empty:
-    print("\nSample rows with missing values:\n", missing_rows.head())
-
-# Sample duplicate rows
-duplicates = df[df.duplicated(keep=False)]
-if not duplicates.empty:
-    print("\nSample duplicate rows:\n", duplicates.head())
-
-# -----------------------------
-# Drug/Crime-related stats
-# -----------------------------
-for col in ["is_drug_related", "is_crime_related", "risk_level"]:
-    if col in df.columns:
-        print(f"\n=== {col} Distribution ===")
-        print(df[col].value_counts())
-        print("Proportion:\n", round(df[col].value_counts(normalize=True), 4))
-
-# Risk level numeric analysis
-if "risk_level" in df.columns and pd.api.types.is_numeric_dtype(df["risk_level"]):
-    print("\n=== Risk Level Stats ===")
-    print("Average risk:", round(df["risk_level"].mean(), 2))
-    print("Max risk:", df["risk_level"].max())
-    high_risk_count = (df["risk_level"] >= 0.7).sum()  # Threshold
-    print("Number of high-risk items (risk >= 0.7):", high_risk_count)
-
-# -----------------------------
-# Time coverage
-# -----------------------------
-if "datetime" in df.columns:
-    df["datetime"] = pd.to_datetime(df["datetime"], errors="coerce")
-    print("\n=== Date Range ===")
-    print("Earliest:", df["datetime"].min())
-    print("Latest:", df["datetime"].max())
-
-    # Daily counts
-    df["date"] = df["datetime"].dt.date
-    daily_counts = df.groupby("date").size()
-    print("\n=== Daily Counts of Posts ===")
-    print(daily_counts)
-
-# -----------------------------
-# Text Analysis
-# -----------------------------
-if "text" in df.columns:
-    df["text"] = df["text"].astype(str)
-    df["text_length"] = df["text"].apply(len)
-    print("\n=== Text Length Stats ===")
-    print("Average length:", round(df["text_length"].mean(), 2))
-    print("Min length:", df["text_length"].min())
-    print("Max length:", df["text_length"].max())
-
-    # Top 10 most common words
-    words = Counter()
-    for t in df["text"]:
-        words.update(re.findall(r"\w+", t.lower()))
-    print("\nTop 10 common words:", words.most_common(10))
-
-# -----------------------------
-# User / Source Analysis
-# -----------------------------
-if "username" in df.columns:
-    print("\n=== User Analysis ===")
-    print("Total unique users:", df["username"].nunique())
-    top_users = df["username"].value_counts().head(10)
-    print("Top 10 users by post count:\n", top_users)
-
-# -----------------------------
-# Scraper Evaluation Metrics
-# -----------------------------
-print("\n=== Scraper Evaluation Metrics ===")
-
-# 1. Completeness (% of filled cells)
-completeness = 1 - df.isna().mean().mean()
-print(f"Completeness (all columns filled): {round(completeness*100, 2)}%")
-
-# 2. Duplicate rate (% of duplicate rows)
-duplicate_rate = df.duplicated().mean()
-print(f"Duplicate rows rate: {round(duplicate_rate*100, 2)}%")
-
-# 3. Drug/Crime relevance (if available)
-for col in ["is_drug_related", "is_crime_related"]:
-    if col in df.columns:
-        relevance = df[col].sum() / len(df)
-        print(f"{col} relevance rate: {round(relevance*100,2)}%")
-
-# 4. Time coverage (active days vs total days)
-if "datetime" in df.columns:
-    total_days = (df["datetime"].max() - df["datetime"].min()).days + 1
-    active_days = df["date"].nunique()
-    coverage_ratio = active_days / total_days
-    print(f"Time coverage ratio (active days / total days): {round(coverage_ratio*100,2)}%")
-
-# 5. Average text length (proxy for content richness)
-if "text" in df.columns:
-    print(f"Average text length: {round(df['text_length'].mean(),2)} characters")
-
-# 6. Classification Metrics (using scraper labels as pseudo-ground truth)
-# If multiple columns available (e.g., is_drug_related vs is_crime_related), compute metrics
-if "is_drug_related" in df.columns and "is_crime_related" in df.columns:
-    y_true = df["is_crime_related"]
-    y_pred = df["is_drug_related"]
-    print("\n=== Classification Metrics (is_drug_related vs is_crime_related) ===")
-    print("Accuracy:", round(accuracy_score(y_true, y_pred), 4))
-    print("Precision:", round(precision_score(y_true, y_pred), 4))
-    print("Recall:", round(recall_score(y_true, y_pred), 4))
-    print("F1-score:", round(f1_score(y_true, y_pred), 4))
-    print("\nClassification Report:\n", classification_report(y_true, y_pred))
-else:
-    print("\n⚠️ Skipping classification metrics: Not enough columns for evaluation.")
-
-print("\n✅ Data evaluation + metrics complete!")