""" AI Vidya — Daily Report Evaluation Agent ========================================= Reads #ai-vidya on Slack, checks who posted an EOD report today, evaluates quality with Claude, posts a summary to #intern-updates. Deploy on HuggingFace Spaces (Gradio SDK) — runs as a persistent web app. Trigger manually via the UI or set a scheduled run via HF ZeroGPU / cron. """ import os import json import time import datetime from zoneinfo import ZoneInfo import anthropic import gradio as gr from slack_sdk import WebClient from slack_sdk.errors import SlackApiError # ─── CONFIG ─────────────────────────────────────────────────────────────────── REPORT_CHANNEL_ID = "C08L1CFEF40" # #ai-vidya ALERT_CHANNEL_ID = "C0B2L4FTW4E" # #intern-updates IST = ZoneInfo("Asia/Kolkata") DEADLINE_HOUR = 23 # 11 PM IST # Intern registry — add/remove interns here # Format: { "slack_user_id": "Full Name" } INTERN_REGISTRY = { "U084C0HQLKS": "Khushi Mishra", "U0AFNM2H88G": "Roshesh Shah", "U0AEQV8F7LM": "Rishikesh Brahma", "U0AEN15QY0K": "Vishwajeet Jadhav", } # ─── CLIENTS (lazy-init from env) ───────────────────────────────────────────── def get_slack(): token = os.environ.get("SLACK_BOT_TOKEN") if not token: raise ValueError("SLACK_BOT_TOKEN not set in environment secrets.") return WebClient(token=token) def get_anthropic(): key = os.environ.get("ANTHROPIC_API_KEY") if not key: raise ValueError("ANTHROPIC_API_KEY not set in environment secrets.") return anthropic.Anthropic(api_key=key) # ─── STEP 1: FETCH TODAY'S MESSAGES ─────────────────────────────────────────── def fetch_todays_messages(slack: WebClient, log: list) -> list: """Return all non-bot messages posted in #ai-vidya today (IST).""" now_ist = datetime.datetime.now(IST) start_of_day = now_ist.replace(hour=0, minute=0, second=0, microsecond=0) oldest_ts = str(start_of_day.timestamp()) log.append(f"📥 Fetching messages from #ai-vidya since {start_of_day.strftime('%d %b %Y %I:%M %p IST')}...") messages = [] cursor = None while True: kwargs = { "channel": REPORT_CHANNEL_ID, "oldest": oldest_ts, "limit": 200, } if cursor: kwargs["cursor"] = cursor resp = slack.conversations_history(**kwargs) batch = [m for m in resp.get("messages", []) if not m.get("bot_id") and not m.get("subtype")] messages.extend(batch) if resp.get("has_more") and resp.get("response_metadata", {}).get("next_cursor"): cursor = resp["response_metadata"]["next_cursor"] else: break log.append(f" → Found {len(messages)} human messages today.") return messages # ─── STEP 2B: CHECK LEAVE STATUS VIA SLACK PROFILE ────────────────────────── def is_on_leave_today(name: str, full_status: str, claude_client) -> bool: """Use Claude to determine if intern is on leave today from their full Slack status string.""" if not full_status or full_status.strip() == "": return False today = datetime.datetime.now(IST).strftime("%d-%m-%Y") prompt = f"""Today is {today}. Intern Slack status: "{full_status}" Is this intern on leave TODAY? Consider: date ranges, "on leave", "off", "absent", :red_circle: emoji, away messages. Reply ONLY with "yes" or "no".""" try: resp = claude_client.messages.create( model="claude-sonnet-4-20250514", max_tokens=5, messages=[{"role": "user", "content": prompt}] ) return resp.content[0].text.strip().lower().startswith("yes") except Exception: return False def get_interns_on_leave(slack: WebClient, claude_client, interns: list, log: list) -> set: """Check each intern's Slack status using Claude to interpret leave intelligently.""" on_leave = set() today = datetime.datetime.now(IST).strftime("%d %b %Y") log.append(f" Checking leave status for {today}...") for intern in interns: try: # Try users.info first resp = slack.users_info(user=intern["userId"]) profile = resp.get("user", {}).get("profile", {}) status_text = profile.get("status_text", "") status_emoji = profile.get("status_emoji", "") full_status = f"{status_emoji} {status_text}".strip() # Fallback: try users_profile_get if status is empty if not full_status: resp2 = slack.users_profile_get(user=intern["userId"]) p2 = resp2.get("profile", {}) status_text = p2.get("status_text", "") status_emoji = p2.get("status_emoji", "") full_status = f"{status_emoji} {status_text}".strip() log.append(f" {intern['name']} status: '{full_status}'") if not full_status: log.append(f" {intern['name']} — no status set, treating as active") continue on_leave_today = is_on_leave_today(intern["name"], full_status, claude_client) if on_leave_today: on_leave.add(intern["userId"]) log.append(f" {intern['name']} — MARKED ON LEAVE") else: log.append(f" {intern['name']} — active") except Exception as e: log.append(f" Could not check {intern['name']}: {e}") return on_leave # ─── STEP 2: MATCH MESSAGES TO INTERNS ──────────────────────────────────────── def match_reports(messages: list, log: list, on_leave: set = None) -> dict: """ Returns dict keyed by user_id: { "U123": { "name": "Priya Sharma", "submitted": True/False, "message": "...", "miss_count": 0 } } """ # Build a set of user_ids who posted today posted_today = {} for msg in messages: uid = msg.get("user") if uid and uid not in posted_today: posted_today[uid] = msg.get("text", "") results = {} # Check registered interns registry = INTERN_REGISTRY if not registry: # Auto-detect from today's posters if registry is empty (fallback) log.append("⚠️ INTERN_REGISTRY is empty — showing all posters instead.") for uid, text in posted_today.items(): results[uid] = { "name": uid, "submitted": True, "message": text, } return results on_leave = on_leave or set() for uid, name in registry.items(): if uid in on_leave: results[uid] = { "name": name, "submitted": True, "message": None, "on_leave": True, } log.append(f" {name} — on leave (skipped)") elif uid in posted_today: results[uid] = { "name": name, "submitted": True, "message": posted_today[uid], "on_leave": False, } log.append(f" {name} — submitted") else: results[uid] = { "name": name, "submitted": False, "message": None, "on_leave": False, } log.append(f" {name} — MISSED") return results # ─── STEP 3: EVALUATE REPORT QUALITY WITH CLAUDE ───────────────────────────── EVAL_SYSTEM = """You evaluate daily EOD reports from AI developer interns. A valid report must contain ALL three: 1. What they actually worked on — specific and technical 2. What they got stuck on — an honest blocker, not vague 3. What they tried to resolve it Return ONLY a JSON object (no markdown, no preamble): { "quality": "good" | "weak" | "invalid", "score": 1-10, "reason": "one sentence max", "flags": [] } Quality: - "good" → specific technical content, real blocker, shows thinking (score 7-10) - "weak" → too vague, missing blocker, just a status update (score 4-6) - "invalid" → one liner, no content, "done", "working on it", link only (score 1-3) Flags (add any that apply): "no_blocker", "vague_progress", "no_technical_detail", "link_only", "too_short" """ def evaluate_report(client: anthropic.Anthropic, name: str, text: str, log: list) -> dict: """Call Claude to evaluate a single report. Returns eval dict.""" if not text or len(text.strip()) < 20: return {"quality": "invalid", "score": 1, "reason": "Report is empty or too short.", "flags": ["too_short"]} try: response = client.messages.create( model="claude-sonnet-4-20250514", max_tokens=300, system=EVAL_SYSTEM, messages=[{"role": "user", "content": f"Intern: {name}\n\nReport:\n{text}"}], ) raw = response.content[0].text.strip().replace("```json", "").replace("```", "") return json.loads(raw) except Exception as e: log.append(f" ⚠️ Claude eval failed for {name}: {e}") return {"quality": "weak", "score": 3, "reason": "Evaluation error.", "flags": []} # ─── STEP 4: GET OR CREATE ALERT CHANNEL ───────────────────────────────────── def get_alert_channel_id(slack: WebClient, log: list) -> str: """Find #intern-updates channel ID - paginate through all channels.""" all_names = [] cursor = None while True: kwargs = dict(types="public_channel,private_channel", limit=200) if cursor: kwargs["cursor"] = cursor resp = slack.conversations_list(**kwargs) for ch in resp.get("channels", []): all_names.append(ch["name"]) if ch["name"] == ALERT_CHANNEL_ID: log.append(f" Found #{ALERT_CHANNEL_ID} (ID: {ch['id']})") return ch["id"] next_cursor = resp.get("response_metadata", {}).get("next_cursor") if not next_cursor: break cursor = next_cursor log.append(f"Channel #{ALERT_CHANNEL_ID} not found. Bot can see: {all_names}") log.append(f" Run in #intern-updates: /invite @intern-eod") return None # ─── STEP 5: BUILD SLACK MESSAGE ────────────────────────────────────────────── def build_slack_message(results: dict, evals: dict, run_time: str) -> str: active = {uid: r for uid, r in results.items() if not r.get("on_leave")} submitted = [r for r in active.values() if r["submitted"]] missed = [r for r in active.values() if not r["submitted"]] on_leave_list = [r for r in results.values() if r.get("on_leave")] good = [r for uid, r in active.items() if r["submitted"] and evals.get(uid, {}).get("quality") == "good"] weak = [r for uid, r in active.items() if r["submitted"] and evals.get(uid, {}).get("quality") == "weak"] invalid = [r for uid, r in active.items() if r["submitted"] and evals.get(uid, {}).get("quality") == "invalid"] total = len(results) active_count = len(active) date_str = datetime.datetime.now(IST).strftime("%d %b %Y") lines = [] lines.append(f"") lines.append(f"*Daily Report Check — {date_str}*") lines.append(f"{'─' * 32}") lines.append(f":busts_in_silhouette: *{active_count} active* · :spiral_calendar_pad: Deadline 11:00 PM IST") lines.append("") # Submitted if submitted: lines.append(f":notepad_spiral: *Reports Received — {len(submitted)}/{active_count}*") for uid, r in active.items(): if not r["submitted"]: continue ev = evals.get(uid, {}) q = ev.get("quality", "?") sc = ev.get("score", "?") icon = {"good": ":large_green_circle:", "weak": ":large_yellow_circle:", "invalid": ":red_circle:"}.get(q, ":white_circle:") flags = ev.get("flags", []) line = f" {icon} *{r['name']}* · {sc}/10 · {ev.get('reason', '')}" if flags: line += "\n _flags: " + ", ".join(flags) + "_" lines.append(line) lines.append("") # Missed if missed: lines.append(f":x: *No Report — {len(missed)}/{active_count}*") for r in missed: lines.append(f" :red_circle: *{r['name']}*") lines.append("") # On Leave if on_leave_list: lines.append(f":palm_tree: *On Leave — {len(on_leave_list)}/{total}*") for r in on_leave_list: lines.append(f" :white_circle: *{r['name']}*") lines.append("") # Footer lines.append(f"{'─' * 32}") lines.append(f":large_green_circle: {len(good)} :large_yellow_circle: {len(weak)} :red_circle: {len(invalid)} :x: {len(missed)} :palm_tree: {len(on_leave_list)}") return "\n".join(lines) # ─── STEP 6: SEND MISSED ALERTS ─────────────────────────────────────────────── def send_missed_alerts(slack: WebClient, alert_channel_id: str, results: dict, log: list): """Post individual @mention alerts for every missed intern — skip interns on leave.""" for uid, r in results.items(): if r["submitted"] or r.get("on_leave"): continue msg = ( f"<@{uid}> Your EOD report for today has not been received.\n\n" f"Reports are expected daily by 11:00 PM IST. " f"If you are blocked, submit what you have worked on and state the blocker — that is a valid submission." ) try: slack.chat_postMessage(channel=alert_channel_id, text=msg) log.append(f" Alert sent for {r['name']}") except SlackApiError as e: log.append(f" Could not send alert for {r['name']}: {e.response['error']}") # ─── MAIN EVALUATION RUNNER ─────────────────────────────────────────────────── def run_evaluation(custom_interns_json: str = "") -> str: """ Full pipeline. Returns a log string for display in Gradio. custom_interns_json: optional JSON override for INTERN_REGISTRY, e.g. {"U06T83LP86P": "Mann Vishnoi", "U084C0HQLKS": "Khushi Mishra"} """ log = [] now_ist = datetime.datetime.now(IST) run_time = now_ist.strftime("%d %b %Y %I:%M %p IST") log.append(f"🚀 Evaluation started at {run_time}") log.append("=" * 55) # Override intern registry if provided via UI global INTERN_REGISTRY if custom_interns_json.strip(): try: INTERN_REGISTRY = json.loads(custom_interns_json) log.append(f"👥 Using custom intern list: {list(INTERN_REGISTRY.values())}") except json.JSONDecodeError: log.append("⚠️ Invalid JSON for intern list — using default INTERN_REGISTRY.") try: slack_client = get_slack() claude_client = get_anthropic() except ValueError as e: log.append(f"❌ {e}") return "\n".join(log) # 1. Fetch messages try: messages = fetch_todays_messages(slack_client, log) except SlackApiError as e: log.append(f"❌ Slack error: {e.response['error']}") return "\n".join(log) # 2. Check who is on leave via Slack status log.append("\nChecking Slack status for leave...") intern_list = [{"userId": uid, "name": name} for uid, name in INTERN_REGISTRY.items()] on_leave = get_interns_on_leave(slack_client, claude_client, intern_list, log) if not on_leave: log.append(" No interns on leave today.") # 3. Match to interns results = match_reports(messages, log, on_leave) if not results: log.append("⚠️ No interns to evaluate.") return "\n".join(log) # 3. Evaluate quality with Claude log.append("\n🤖 Evaluating report quality with Claude...") evals = {} for uid, r in results.items(): if r["submitted"] and not r.get("on_leave"): ev = evaluate_report(claude_client, r["name"], r["message"], log) evals[uid] = ev q_icon = {"good": "🟢", "weak": "🟡", "invalid": "🔴"}.get(ev["quality"], "⚪") log.append(f" {q_icon} {r['name']} → {ev['quality']} (score {ev['score']}/10): {ev['reason']}") # 4. Alert channel is hardcoded - no API call needed alert_channel_id = ALERT_CHANNEL_ID log.append(f" Using #intern-updates ({ALERT_CHANNEL_ID})") # 5. Build and post full message to #ai-vidya only summary_msg = build_slack_message(results, evals, run_time) try: slack_client.chat_postMessage(channel=REPORT_CHANNEL_ID, text=summary_msg) log.append(" Summary posted to #ai-vidya.") except SlackApiError as e: log.append(f" Could not post to #ai-vidya: {e.response['error']}") # 6. Send @mention alerts to #ai-vidya missed_count = sum(1 for r in results.values() if not r["submitted"] and not r.get("on_leave")) if missed_count: log.append(f"\nSending {missed_count} missed report alert(s) to #ai-vidya...") send_missed_alerts(slack_client, REPORT_CHANNEL_ID, results, log) else: log.append("\nAll interns submitted — no missed alerts needed.") log.append("\n" + "=" * 55) log.append("✅ Evaluation complete.") return "\n".join(log) # ─── GRADIO UI ──────────────────────────────────────────────────────────────── INTERN_REGISTRY_EXAMPLE = json.dumps({ "U06T83LP86P": "Mann Vishnoi", "U084C0HQLKS": "Khushi Mishra", "U0AFNM2H88G": "Roshesh Shah", }, indent=2) with gr.Blocks(title="AI Vidya — Daily Report Agent", theme=gr.themes.Soft()) as demo: gr.Markdown(""" # 📋 AI Vidya — Daily Report Evaluation Agent Reads **#ai-vidya**, checks who submitted their EOD report today, evaluates quality with Claude, and posts results to **#intern-updates**. --- **Before running:** Add `SLACK_BOT_TOKEN` and `ANTHROPIC_API_KEY` in *Settings → Secrets* of this HuggingFace Space. """) with gr.Row(): with gr.Column(scale=1): gr.Markdown("### 👥 Intern Registry") intern_input = gr.Textbox( label="Intern list (JSON: {slack_user_id: name})", value=INTERN_REGISTRY_EXAMPLE, lines=10, placeholder='{"U06T83LP86P": "Mann Vishnoi"}', info="Get Slack user IDs from: Slack profile → ⋮ menu → Copy member ID" ) run_btn = gr.Button("▶️ Run Evaluation Now", variant="primary", size="lg") with gr.Column(scale=2): gr.Markdown("### 📄 Evaluation Log") output = gr.Textbox( label="", lines=28, interactive=False, show_copy_button=True, ) gr.Markdown(""" --- ### 🕐 Auto-scheduling on HuggingFace HuggingFace Spaces don't have built-in cron. Two options: **Option A — GitHub Actions (recommended, free)** Create `.github/workflows/trigger.yml` in your HF Space repo: ```yaml on: schedule: - cron: '30 17 * * *' # 11:00 PM IST = 17:30 UTC jobs: trigger: runs-on: ubuntu-latest steps: - run: curl -X POST ${{ secrets.HF_SPACE_URL }}/run/predict -H "Content-Type: application/json" -d '{}' ``` **Option B — External cron service** Use [cron-job.org](https://cron-job.org) (free) to POST to your Space URL daily at 11 PM IST. """) run_btn.click(fn=run_evaluation, inputs=[intern_input], outputs=[output]) # ─── CRON TRIGGER — runs evaluation synchronously and returns result ────────── import threading def trigger_evaluation(): """Called by the /trigger Gradio endpoint — runs fully, returns log.""" return run_evaluation("") # Expose as a hidden Gradio endpoint so cron can call it with demo: cron_output = gr.Textbox(visible=False, label="cron_result") cron_btn = gr.Button(visible=False, elem_id="cron_trigger") cron_btn.click(fn=trigger_evaluation, inputs=[], outputs=[cron_output], api_name="trigger") if __name__ == "__main__": demo.launch()