Spaces:
Sleeping
Sleeping
refactor: unify recent-activity logic with generic engine + processors
Browse files
app.py
CHANGED
|
@@ -58,6 +58,9 @@ def build_session(accept: str) -> requests.Session:
|
|
| 58 |
return session
|
| 59 |
|
| 60 |
|
|
|
|
|
|
|
|
|
|
| 61 |
def normalize_recent_activity_payload(payload):
|
| 62 |
if isinstance(payload, list):
|
| 63 |
return payload, None
|
|
@@ -76,15 +79,9 @@ def normalize_recent_activity_payload(payload):
|
|
| 76 |
|
| 77 |
|
| 78 |
def get_year_bounds(selected_year: int):
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
return
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
def should_stop_after_crossing_year_start(
|
| 85 |
-
crossed_before_year_start: bool,
|
| 86 |
-
) -> bool:
|
| 87 |
-
return crossed_before_year_start
|
| 88 |
|
| 89 |
|
| 90 |
def build_events_page_signature(events: list[dict]) -> tuple:
|
|
@@ -104,6 +101,71 @@ def build_events_page_signature(events: list[dict]) -> tuple:
|
|
| 104 |
return tuple(signature)
|
| 105 |
|
| 106 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
# ============================================================
|
| 108 |
# Thread-safe rate limiters
|
| 109 |
# ============================================================
|
|
@@ -485,7 +547,7 @@ def build_invalid_section_state(username: str, selected_year: int, empty_state_f
|
|
| 485 |
|
| 486 |
|
| 487 |
# ============================================================
|
| 488 |
-
# Community activity
|
| 489 |
# ============================================================
|
| 490 |
def empty_community_state(selected_year: int):
|
| 491 |
return {
|
|
@@ -500,36 +562,30 @@ def empty_community_state(selected_year: int):
|
|
| 500 |
}
|
| 501 |
|
| 502 |
|
| 503 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 504 |
username: str,
|
| 505 |
selected_year: int,
|
| 506 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 507 |
):
|
| 508 |
username = (username or "").strip()
|
| 509 |
selected_year = int(selected_year)
|
| 510 |
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
"thread_opened": "Community Threads",
|
| 515 |
-
"discussion_activity": "Community Activity",
|
| 516 |
-
}
|
| 517 |
-
title = title_map.get(mode, "Community")
|
| 518 |
-
|
| 519 |
-
if not username:
|
| 520 |
-
return {
|
| 521 |
-
"title": title,
|
| 522 |
-
"fetched": True,
|
| 523 |
-
"dates": [],
|
| 524 |
-
"count": 0,
|
| 525 |
-
"logs": "No username provided.",
|
| 526 |
-
}
|
| 527 |
|
| 528 |
-
dates = []
|
| 529 |
-
seen_ids = set()
|
| 530 |
-
skip = 0
|
| 531 |
-
limit = RECENT_ACTIVITY_LIMIT
|
| 532 |
cursor = None
|
|
|
|
| 533 |
pages_fetched = 0
|
| 534 |
|
| 535 |
seen_target_year = False
|
|
@@ -540,93 +596,19 @@ def fetch_user_discussion_activity(
|
|
| 540 |
seen_cursors = set()
|
| 541 |
seen_page_signatures = set()
|
| 542 |
|
| 543 |
-
OPEN_EVENT_EPSILON_SECONDS = 90
|
| 544 |
logs = []
|
| 545 |
|
| 546 |
-
year_start, year_end = get_year_bounds(selected_year)
|
| 547 |
-
session = build_session("application/json")
|
| 548 |
-
|
| 549 |
-
def extract_author_name(discussion: dict) -> str | None:
|
| 550 |
-
author = discussion.get("author")
|
| 551 |
-
if isinstance(author, str):
|
| 552 |
-
return author
|
| 553 |
-
if isinstance(author, dict):
|
| 554 |
-
for key in ["name", "username", "login"]:
|
| 555 |
-
value = author.get(key)
|
| 556 |
-
if isinstance(value, str):
|
| 557 |
-
return value
|
| 558 |
-
return None
|
| 559 |
-
|
| 560 |
-
def parse_ts(raw_value):
|
| 561 |
-
if not raw_value:
|
| 562 |
-
return None
|
| 563 |
-
try:
|
| 564 |
-
return pd.to_datetime(raw_value, utc=True)
|
| 565 |
-
except Exception:
|
| 566 |
-
return None
|
| 567 |
-
|
| 568 |
-
def event_matches_mode(event: dict, discussion: dict) -> tuple[bool, str | None]:
|
| 569 |
-
is_pr = bool(discussion.get("isPullRequest", False))
|
| 570 |
-
author_name = extract_author_name(discussion)
|
| 571 |
-
repo_id = event.get("repoId", "unknown-repo")
|
| 572 |
-
discussion_num = discussion.get("num", "unknown-num")
|
| 573 |
-
event_id = event.get("eventId", "unknown-event")
|
| 574 |
-
|
| 575 |
-
if mode == "pr_opened":
|
| 576 |
-
if not is_pr:
|
| 577 |
-
return False, None
|
| 578 |
-
if not author_name or author_name.lower() != username.lower():
|
| 579 |
-
return False, None
|
| 580 |
-
return True, f"{repo_id}::pr::{discussion_num}"
|
| 581 |
-
|
| 582 |
-
if mode == "pr_activity":
|
| 583 |
-
if not is_pr:
|
| 584 |
-
return False, None
|
| 585 |
-
return True, f"event::{event_id}"
|
| 586 |
-
|
| 587 |
-
if mode == "thread_opened":
|
| 588 |
-
if is_pr:
|
| 589 |
-
return False, None
|
| 590 |
-
if not author_name or author_name.lower() != username.lower():
|
| 591 |
-
return False, None
|
| 592 |
-
return True, f"{repo_id}::thread::{discussion_num}"
|
| 593 |
-
|
| 594 |
-
if mode == "discussion_activity":
|
| 595 |
-
if is_pr:
|
| 596 |
-
return False, None
|
| 597 |
-
return True, f"event::{event_id}"
|
| 598 |
-
|
| 599 |
-
return False, None
|
| 600 |
-
|
| 601 |
-
def extract_timestamp_for_mode(event: dict, discussion: dict):
|
| 602 |
-
event_ts = parse_ts(event.get("time"))
|
| 603 |
-
created_ts = parse_ts(discussion.get("createdAt"))
|
| 604 |
-
|
| 605 |
-
if mode in {"pr_opened", "thread_opened"}:
|
| 606 |
-
return created_ts
|
| 607 |
-
|
| 608 |
-
if mode in {"pr_activity", "discussion_activity"}:
|
| 609 |
-
if event_ts is None:
|
| 610 |
-
return None
|
| 611 |
-
if created_ts is not None:
|
| 612 |
-
delta_seconds = abs((event_ts - created_ts).total_seconds())
|
| 613 |
-
if delta_seconds <= OPEN_EVENT_EPSILON_SECONDS:
|
| 614 |
-
return None
|
| 615 |
-
return event_ts
|
| 616 |
-
|
| 617 |
-
return None
|
| 618 |
-
|
| 619 |
try:
|
| 620 |
while True:
|
| 621 |
-
if pages_fetched >=
|
| 622 |
logs.append("Stopped due to safety max pages.")
|
| 623 |
break
|
| 624 |
|
| 625 |
-
|
| 626 |
|
| 627 |
params = {
|
| 628 |
"limit": limit,
|
| 629 |
-
"activityType":
|
| 630 |
"feedType": "user",
|
| 631 |
"entity": username,
|
| 632 |
}
|
|
@@ -669,313 +651,330 @@ def fetch_user_discussion_activity(
|
|
| 669 |
if not isinstance(event, dict):
|
| 670 |
continue
|
| 671 |
|
| 672 |
-
event_ts =
|
| 673 |
-
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
|
| 679 |
-
|
| 680 |
-
|
| 681 |
-
|
| 682 |
-
|
| 683 |
-
|
| 684 |
-
|
| 685 |
-
|
| 686 |
-
|
| 687 |
-
|
| 688 |
-
|
| 689 |
-
|
| 690 |
-
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
elif dt < year_start:
|
| 699 |
-
continue
|
| 700 |
-
elif dt > year_end:
|
| 701 |
-
continue
|
| 702 |
-
|
| 703 |
-
if unique_id in seen_ids:
|
| 704 |
-
continue
|
| 705 |
-
|
| 706 |
-
seen_ids.add(unique_id)
|
| 707 |
-
dates.append(dt.tz_localize(None).date())
|
| 708 |
-
|
| 709 |
-
if should_stop_after_crossing_year_start(crossed_before_year_start):
|
| 710 |
logs.append("Stopped after crossing before selected year.")
|
| 711 |
break
|
| 712 |
|
| 713 |
-
|
| 714 |
-
|
| 715 |
-
|
| 716 |
-
|
| 717 |
-
|
| 718 |
|
| 719 |
-
if stalled_oldest_count >=
|
| 720 |
logs.append("Stopped because pagination no longer progressed toward older events.")
|
| 721 |
break
|
| 722 |
|
| 723 |
-
if page_oldest_event_ts is not None:
|
| 724 |
-
previous_oldest_event_ts_seen = page_oldest_event_ts
|
| 725 |
-
|
| 726 |
if next_cursor:
|
| 727 |
cursor = next_cursor
|
| 728 |
else:
|
| 729 |
skip += limit
|
| 730 |
|
| 731 |
-
|
| 732 |
-
|
| 733 |
-
|
| 734 |
-
|
| 735 |
-
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
| 741 |
-
"dates": dates,
|
| 742 |
-
"count": len(dates),
|
| 743 |
-
"logs": "\n".join(logs),
|
| 744 |
-
}
|
| 745 |
|
| 746 |
except Exception as e:
|
| 747 |
-
return
|
| 748 |
-
|
| 749 |
-
|
| 750 |
-
|
| 751 |
-
|
| 752 |
-
|
| 753 |
-
}
|
| 754 |
-
|
| 755 |
-
|
| 756 |
-
def fetch_all_community_activity(username: str, selected_year: int):
|
| 757 |
-
username = (username or "").strip()
|
| 758 |
-
selected_year = int(selected_year)
|
| 759 |
-
|
| 760 |
-
cache_key = (username.lower(), selected_year, "community")
|
| 761 |
-
cached = get_cache(cache_key)
|
| 762 |
-
if cached:
|
| 763 |
-
return cached
|
| 764 |
-
|
| 765 |
-
if not username:
|
| 766 |
-
state = empty_community_state(selected_year)
|
| 767 |
-
state["fetched"] = True
|
| 768 |
-
state["logs"] = "No username provided."
|
| 769 |
-
set_cache(cache_key, state)
|
| 770 |
-
return state
|
| 771 |
-
|
| 772 |
-
pr_opened = fetch_user_discussion_activity(username, selected_year, "pr_opened")
|
| 773 |
-
pr_activity = fetch_user_discussion_activity(username, selected_year, "pr_activity")
|
| 774 |
-
thread_opened = fetch_user_discussion_activity(username, selected_year, "thread_opened")
|
| 775 |
-
discussion_activity = fetch_user_discussion_activity(username, selected_year, "discussion_activity")
|
| 776 |
-
|
| 777 |
-
combined_logs = "\n\n".join(
|
| 778 |
-
[
|
| 779 |
-
"[Pull Requests]\n" + pr_opened["logs"],
|
| 780 |
-
"[PR Activity]\n" + pr_activity["logs"],
|
| 781 |
-
"[Community Threads]\n" + thread_opened["logs"],
|
| 782 |
-
"[Community Activity]\n" + discussion_activity["logs"],
|
| 783 |
-
]
|
| 784 |
-
)
|
| 785 |
-
|
| 786 |
-
result = {
|
| 787 |
-
"username": username,
|
| 788 |
-
"selected_year": selected_year,
|
| 789 |
-
"fetched": True,
|
| 790 |
-
"pr_opened": pr_opened,
|
| 791 |
-
"pr_activity": pr_activity,
|
| 792 |
-
"thread_opened": thread_opened,
|
| 793 |
-
"discussion_activity": discussion_activity,
|
| 794 |
-
"logs": combined_logs,
|
| 795 |
-
}
|
| 796 |
-
|
| 797 |
-
set_cache(cache_key, result)
|
| 798 |
-
return result
|
| 799 |
|
| 800 |
|
| 801 |
# ============================================================
|
| 802 |
-
#
|
| 803 |
# ============================================================
|
| 804 |
-
|
| 805 |
-
|
| 806 |
-
|
| 807 |
-
|
| 808 |
-
|
| 809 |
-
|
| 810 |
-
|
| 811 |
-
|
| 812 |
-
|
| 813 |
-
|
| 814 |
-
|
| 815 |
-
|
| 816 |
-
|
| 817 |
-
|
| 818 |
-
cached = get_cache(cache_key)
|
| 819 |
-
if cached:
|
| 820 |
-
return cached
|
| 821 |
-
|
| 822 |
-
if not username:
|
| 823 |
-
state = empty_bucketed_activity_state(selected_year, empty_logs_text, bucket_titles)
|
| 824 |
-
state["fetched"] = True
|
| 825 |
-
state["logs"] = "No username provided."
|
| 826 |
-
set_cache(cache_key, state)
|
| 827 |
-
return state
|
| 828 |
-
|
| 829 |
-
bucket_dates = {key: [] for key in bucket_titles}
|
| 830 |
-
bucket_seen_ids = {key: set() for key in bucket_titles}
|
| 831 |
-
|
| 832 |
-
cursor = None
|
| 833 |
-
skip = 0
|
| 834 |
-
limit = RECENT_ACTIVITY_LIMIT
|
| 835 |
-
pages_fetched = 0
|
| 836 |
-
|
| 837 |
-
seen_target_year = False
|
| 838 |
-
crossed_before_year_start = False
|
| 839 |
-
oldest_event_ts_seen = None
|
| 840 |
-
previous_oldest_event_ts_seen = None
|
| 841 |
-
stalled_oldest_count = 0
|
| 842 |
-
seen_cursors = set()
|
| 843 |
-
seen_page_signatures = set()
|
| 844 |
-
|
| 845 |
-
logs = []
|
| 846 |
-
|
| 847 |
-
year_start, year_end = get_year_bounds(selected_year)
|
| 848 |
-
session = build_session("application/json")
|
| 849 |
-
|
| 850 |
-
try:
|
| 851 |
-
while True:
|
| 852 |
-
if pages_fetched >= RECENT_ACTIVITY_MAX_PAGES:
|
| 853 |
-
logs.append("Stopped due to safety max pages.")
|
| 854 |
-
break
|
| 855 |
-
|
| 856 |
-
recent_activity_rate_limiter.wait()
|
| 857 |
-
|
| 858 |
-
params = {
|
| 859 |
-
"limit": limit,
|
| 860 |
-
"activityType": activity_type,
|
| 861 |
-
"feedType": "user",
|
| 862 |
-
"entity": username,
|
| 863 |
-
}
|
| 864 |
-
if cursor:
|
| 865 |
-
params["cursor"] = cursor
|
| 866 |
-
else:
|
| 867 |
-
params["skip"] = skip
|
| 868 |
-
|
| 869 |
-
response = session.get(
|
| 870 |
-
"https://huggingface.co/api/recent-activity",
|
| 871 |
-
params=params,
|
| 872 |
-
timeout=20,
|
| 873 |
-
)
|
| 874 |
-
response.raise_for_status()
|
| 875 |
-
|
| 876 |
-
payload = response.json()
|
| 877 |
-
pages_fetched += 1
|
| 878 |
|
| 879 |
-
|
|
|
|
| 880 |
|
| 881 |
-
|
| 882 |
-
|
| 883 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 884 |
|
| 885 |
-
|
| 886 |
-
|
| 887 |
-
|
| 888 |
-
|
| 889 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 890 |
|
| 891 |
-
|
| 892 |
-
|
| 893 |
-
|
| 894 |
-
|
| 895 |
-
|
|
|
|
|
|
|
| 896 |
|
| 897 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 898 |
|
| 899 |
-
|
| 900 |
-
|
| 901 |
-
|
|
|
|
| 902 |
|
| 903 |
-
|
| 904 |
-
|
| 905 |
-
|
| 906 |
|
| 907 |
-
|
| 908 |
-
|
| 909 |
-
|
| 910 |
-
continue
|
| 911 |
|
| 912 |
-
|
| 913 |
-
|
| 914 |
-
if page_oldest_event_ts is None or event_ts < page_oldest_event_ts:
|
| 915 |
-
page_oldest_event_ts = event_ts
|
| 916 |
|
| 917 |
-
|
| 918 |
-
|
| 919 |
-
|
| 920 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 921 |
|
| 922 |
-
|
| 923 |
-
continue
|
| 924 |
|
| 925 |
-
|
| 926 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 927 |
|
| 928 |
-
|
|
|
|
|
|
|
| 929 |
|
| 930 |
-
|
| 931 |
-
|
|
|
|
| 932 |
|
| 933 |
-
|
| 934 |
-
|
|
|
|
| 935 |
|
| 936 |
-
|
| 937 |
-
|
| 938 |
|
| 939 |
-
|
| 940 |
-
|
| 941 |
|
| 942 |
-
|
| 943 |
-
|
| 944 |
-
break
|
| 945 |
|
| 946 |
-
|
| 947 |
-
|
| 948 |
-
|
| 949 |
-
|
| 950 |
-
|
|
|
|
|
|
|
|
|
|
| 951 |
|
| 952 |
-
|
| 953 |
-
|
| 954 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 955 |
|
| 956 |
-
|
| 957 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 958 |
|
| 959 |
-
|
| 960 |
-
|
| 961 |
-
|
| 962 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 963 |
|
| 964 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 965 |
"username": username,
|
| 966 |
"selected_year": selected_year,
|
| 967 |
"fetched": True,
|
| 968 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 969 |
}
|
| 970 |
|
| 971 |
-
for key, title in bucket_titles.items():
|
| 972 |
-
result[key] = build_fetched_bucket(
|
| 973 |
-
title=title,
|
| 974 |
-
dates=bucket_dates[key],
|
| 975 |
-
selected_year=selected_year,
|
| 976 |
-
label=bucket_log_labels[key],
|
| 977 |
-
)
|
| 978 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 979 |
logs.append(f"Fetched {pages_fetched} page(s) from recent-activity.")
|
| 980 |
if oldest_event_ts_seen is not None:
|
| 981 |
logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
|
|
@@ -983,30 +982,54 @@ def fetch_bucketed_recent_activity(
|
|
| 983 |
logs.append(f"Crossed before year start: {crossed_before_year_start}.")
|
| 984 |
logs.append(
|
| 985 |
" | ".join(
|
| 986 |
-
f"{bucket_titles[key]}: {len(bucket_dates[key])}"
|
| 987 |
-
for key in bucket_titles
|
| 988 |
)
|
| 989 |
)
|
| 990 |
|
| 991 |
-
result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 992 |
|
| 993 |
-
set_cache(cache_key, result)
|
| 994 |
return result
|
| 995 |
|
| 996 |
-
|
| 997 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 998 |
"username": username,
|
| 999 |
"selected_year": selected_year,
|
| 1000 |
"fetched": True,
|
| 1001 |
-
"logs": f"{
|
| 1002 |
}
|
| 1003 |
|
| 1004 |
-
for key, title in bucket_titles.items():
|
| 1005 |
-
|
| 1006 |
|
| 1007 |
-
return
|
| 1008 |
|
| 1009 |
|
|
|
|
|
|
|
|
|
|
| 1010 |
def classify_upvote_event(event: dict):
|
| 1011 |
target_type = event.get("targetType")
|
| 1012 |
target = event.get("target") or {}
|
|
@@ -1157,19 +1180,76 @@ def empty_single_section_state(section_key: str, selected_year: int):
|
|
| 1157 |
)
|
| 1158 |
|
| 1159 |
|
| 1160 |
-
|
| 1161 |
-
|
| 1162 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1163 |
username=username,
|
| 1164 |
selected_year=selected_year,
|
| 1165 |
-
|
| 1166 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1167 |
bucket_titles=cfg["bucket_titles"],
|
| 1168 |
bucket_log_labels=cfg["bucket_log_labels"],
|
| 1169 |
-
empty_logs_text=cfg["empty_logs_text"],
|
| 1170 |
classifier=cfg["classifier"],
|
|
|
|
| 1171 |
)
|
| 1172 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1173 |
|
| 1174 |
# ============================================================
|
| 1175 |
# Contribution grid helpers
|
|
|
|
| 58 |
return session
|
| 59 |
|
| 60 |
|
| 61 |
+
# ============================================================
|
| 62 |
+
# Recent-activity shared helpers
|
| 63 |
+
# ============================================================
|
| 64 |
def normalize_recent_activity_payload(payload):
|
| 65 |
if isinstance(payload, list):
|
| 66 |
return payload, None
|
|
|
|
| 79 |
|
| 80 |
|
| 81 |
def get_year_bounds(selected_year: int):
|
| 82 |
+
year_start = pd.Timestamp(f"{selected_year}-01-01", tz="UTC")
|
| 83 |
+
year_end = pd.Timestamp(f"{selected_year}-12-31 23:59:59.999999", tz="UTC")
|
| 84 |
+
return year_start, year_end
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
|
| 86 |
|
| 87 |
def build_events_page_signature(events: list[dict]) -> tuple:
|
|
|
|
| 101 |
return tuple(signature)
|
| 102 |
|
| 103 |
|
| 104 |
+
def parse_recent_activity_event_ts(event: dict):
|
| 105 |
+
raw_time = event.get("time")
|
| 106 |
+
if not raw_time:
|
| 107 |
+
return None
|
| 108 |
+
try:
|
| 109 |
+
return pd.to_datetime(raw_time, utc=True)
|
| 110 |
+
except Exception:
|
| 111 |
+
return None
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def update_recent_activity_progress(
|
| 115 |
+
event_ts,
|
| 116 |
+
*,
|
| 117 |
+
year_start,
|
| 118 |
+
year_end,
|
| 119 |
+
oldest_event_ts_seen,
|
| 120 |
+
page_oldest_event_ts,
|
| 121 |
+
seen_target_year,
|
| 122 |
+
crossed_before_year_start,
|
| 123 |
+
):
|
| 124 |
+
if event_ts is None:
|
| 125 |
+
return (
|
| 126 |
+
oldest_event_ts_seen,
|
| 127 |
+
page_oldest_event_ts,
|
| 128 |
+
seen_target_year,
|
| 129 |
+
crossed_before_year_start,
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
if oldest_event_ts_seen is None or event_ts < oldest_event_ts_seen:
|
| 133 |
+
oldest_event_ts_seen = event_ts
|
| 134 |
+
|
| 135 |
+
if page_oldest_event_ts is None or event_ts < page_oldest_event_ts:
|
| 136 |
+
page_oldest_event_ts = event_ts
|
| 137 |
+
|
| 138 |
+
if year_start <= event_ts <= year_end:
|
| 139 |
+
seen_target_year = True
|
| 140 |
+
elif event_ts < year_start:
|
| 141 |
+
crossed_before_year_start = True
|
| 142 |
+
|
| 143 |
+
return (
|
| 144 |
+
oldest_event_ts_seen,
|
| 145 |
+
page_oldest_event_ts,
|
| 146 |
+
seen_target_year,
|
| 147 |
+
crossed_before_year_start,
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def update_stalled_pagination_state(
|
| 152 |
+
*,
|
| 153 |
+
page_oldest_event_ts,
|
| 154 |
+
previous_oldest_event_ts_seen,
|
| 155 |
+
stalled_oldest_count,
|
| 156 |
+
):
|
| 157 |
+
if previous_oldest_event_ts_seen is not None and page_oldest_event_ts is not None:
|
| 158 |
+
if page_oldest_event_ts >= previous_oldest_event_ts_seen:
|
| 159 |
+
stalled_oldest_count += 1
|
| 160 |
+
else:
|
| 161 |
+
stalled_oldest_count = 0
|
| 162 |
+
|
| 163 |
+
if page_oldest_event_ts is not None:
|
| 164 |
+
previous_oldest_event_ts_seen = page_oldest_event_ts
|
| 165 |
+
|
| 166 |
+
return previous_oldest_event_ts_seen, stalled_oldest_count
|
| 167 |
+
|
| 168 |
+
|
| 169 |
# ============================================================
|
| 170 |
# Thread-safe rate limiters
|
| 171 |
# ============================================================
|
|
|
|
| 547 |
|
| 548 |
|
| 549 |
# ============================================================
|
| 550 |
+
# Community activity state
|
| 551 |
# ============================================================
|
| 552 |
def empty_community_state(selected_year: int):
|
| 553 |
return {
|
|
|
|
| 562 |
}
|
| 563 |
|
| 564 |
|
| 565 |
+
# ============================================================
|
| 566 |
+
# Generic recent-activity engine
|
| 567 |
+
# ============================================================
|
| 568 |
+
def collect_recent_activity(
|
| 569 |
+
*,
|
| 570 |
username: str,
|
| 571 |
selected_year: int,
|
| 572 |
+
activity_type: str,
|
| 573 |
+
processor,
|
| 574 |
+
build_session_fn,
|
| 575 |
+
rate_limiter_obj,
|
| 576 |
+
limit: int = RECENT_ACTIVITY_LIMIT,
|
| 577 |
+
max_pages: int = RECENT_ACTIVITY_MAX_PAGES,
|
| 578 |
+
max_stalled_pages: int = RECENT_ACTIVITY_MAX_STALLED_PAGES,
|
| 579 |
):
|
| 580 |
username = (username or "").strip()
|
| 581 |
selected_year = int(selected_year)
|
| 582 |
|
| 583 |
+
processor_state = processor.init_state()
|
| 584 |
+
year_start, year_end = get_year_bounds(selected_year)
|
| 585 |
+
session = build_session_fn("application/json")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 586 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 587 |
cursor = None
|
| 588 |
+
skip = 0
|
| 589 |
pages_fetched = 0
|
| 590 |
|
| 591 |
seen_target_year = False
|
|
|
|
| 596 |
seen_cursors = set()
|
| 597 |
seen_page_signatures = set()
|
| 598 |
|
|
|
|
| 599 |
logs = []
|
| 600 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 601 |
try:
|
| 602 |
while True:
|
| 603 |
+
if pages_fetched >= max_pages:
|
| 604 |
logs.append("Stopped due to safety max pages.")
|
| 605 |
break
|
| 606 |
|
| 607 |
+
rate_limiter_obj.wait()
|
| 608 |
|
| 609 |
params = {
|
| 610 |
"limit": limit,
|
| 611 |
+
"activityType": activity_type,
|
| 612 |
"feedType": "user",
|
| 613 |
"entity": username,
|
| 614 |
}
|
|
|
|
| 651 |
if not isinstance(event, dict):
|
| 652 |
continue
|
| 653 |
|
| 654 |
+
event_ts = parse_recent_activity_event_ts(event)
|
| 655 |
+
|
| 656 |
+
(
|
| 657 |
+
oldest_event_ts_seen,
|
| 658 |
+
page_oldest_event_ts,
|
| 659 |
+
seen_target_year,
|
| 660 |
+
crossed_before_year_start,
|
| 661 |
+
) = update_recent_activity_progress(
|
| 662 |
+
event_ts,
|
| 663 |
+
year_start=year_start,
|
| 664 |
+
year_end=year_end,
|
| 665 |
+
oldest_event_ts_seen=oldest_event_ts_seen,
|
| 666 |
+
page_oldest_event_ts=page_oldest_event_ts,
|
| 667 |
+
seen_target_year=seen_target_year,
|
| 668 |
+
crossed_before_year_start=crossed_before_year_start,
|
| 669 |
+
)
|
| 670 |
+
|
| 671 |
+
processor.process_event(
|
| 672 |
+
event=event,
|
| 673 |
+
event_ts=event_ts,
|
| 674 |
+
year_start=year_start,
|
| 675 |
+
year_end=year_end,
|
| 676 |
+
state=processor_state,
|
| 677 |
+
)
|
| 678 |
+
|
| 679 |
+
if crossed_before_year_start:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 680 |
logs.append("Stopped after crossing before selected year.")
|
| 681 |
break
|
| 682 |
|
| 683 |
+
previous_oldest_event_ts_seen, stalled_oldest_count = update_stalled_pagination_state(
|
| 684 |
+
page_oldest_event_ts=page_oldest_event_ts,
|
| 685 |
+
previous_oldest_event_ts_seen=previous_oldest_event_ts_seen,
|
| 686 |
+
stalled_oldest_count=stalled_oldest_count,
|
| 687 |
+
)
|
| 688 |
|
| 689 |
+
if stalled_oldest_count >= max_stalled_pages:
|
| 690 |
logs.append("Stopped because pagination no longer progressed toward older events.")
|
| 691 |
break
|
| 692 |
|
|
|
|
|
|
|
|
|
|
| 693 |
if next_cursor:
|
| 694 |
cursor = next_cursor
|
| 695 |
else:
|
| 696 |
skip += limit
|
| 697 |
|
| 698 |
+
return processor.build_result(
|
| 699 |
+
username=username,
|
| 700 |
+
selected_year=selected_year,
|
| 701 |
+
state=processor_state,
|
| 702 |
+
logs=logs,
|
| 703 |
+
pages_fetched=pages_fetched,
|
| 704 |
+
oldest_event_ts_seen=oldest_event_ts_seen,
|
| 705 |
+
seen_target_year=seen_target_year,
|
| 706 |
+
crossed_before_year_start=crossed_before_year_start,
|
| 707 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 708 |
|
| 709 |
except Exception as e:
|
| 710 |
+
return processor.build_error_result(
|
| 711 |
+
username=username,
|
| 712 |
+
selected_year=selected_year,
|
| 713 |
+
error=e,
|
| 714 |
+
logs=logs,
|
| 715 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 716 |
|
| 717 |
|
| 718 |
# ============================================================
|
| 719 |
+
# Recent-activity processors
|
| 720 |
# ============================================================
|
| 721 |
+
class CommunityActivityProcessor:
|
| 722 |
+
TITLE_MAP = {
|
| 723 |
+
"pr_opened": "Pull Requests",
|
| 724 |
+
"pr_activity": "PR Activity",
|
| 725 |
+
"thread_opened": "Community Threads",
|
| 726 |
+
"discussion_activity": "Community Activity",
|
| 727 |
+
}
|
| 728 |
+
LOG_LABEL_MAP = {
|
| 729 |
+
"pr_opened": "unique pr_opened event(s)",
|
| 730 |
+
"pr_activity": "unique pr_activity event(s)",
|
| 731 |
+
"thread_opened": "unique thread_opened event(s)",
|
| 732 |
+
"discussion_activity": "unique discussion_activity event(s)",
|
| 733 |
+
}
|
| 734 |
+
OPEN_EVENT_EPSILON_SECONDS = 90
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 735 |
|
| 736 |
+
def __init__(self, username: str):
|
| 737 |
+
self.username = (username or "").strip()
|
| 738 |
|
| 739 |
+
def init_state(self):
|
| 740 |
+
return {
|
| 741 |
+
"dates": {
|
| 742 |
+
"pr_opened": [],
|
| 743 |
+
"pr_activity": [],
|
| 744 |
+
"thread_opened": [],
|
| 745 |
+
"discussion_activity": [],
|
| 746 |
+
},
|
| 747 |
+
"seen_ids": {
|
| 748 |
+
"pr_opened": set(),
|
| 749 |
+
"pr_activity": set(),
|
| 750 |
+
"thread_opened": set(),
|
| 751 |
+
"discussion_activity": set(),
|
| 752 |
+
},
|
| 753 |
+
}
|
| 754 |
|
| 755 |
+
def extract_author_name(self, discussion: dict) -> str | None:
|
| 756 |
+
author = discussion.get("author")
|
| 757 |
+
if isinstance(author, str):
|
| 758 |
+
return author
|
| 759 |
+
if isinstance(author, dict):
|
| 760 |
+
for key in ["name", "username", "login"]:
|
| 761 |
+
value = author.get(key)
|
| 762 |
+
if isinstance(value, str):
|
| 763 |
+
return value
|
| 764 |
+
return None
|
| 765 |
|
| 766 |
+
def parse_ts(self, raw_value):
|
| 767 |
+
if not raw_value:
|
| 768 |
+
return None
|
| 769 |
+
try:
|
| 770 |
+
return pd.to_datetime(raw_value, utc=True)
|
| 771 |
+
except Exception:
|
| 772 |
+
return None
|
| 773 |
|
| 774 |
+
def classify_event(self, event: dict, discussion: dict) -> tuple[str | None, str | None]:
|
| 775 |
+
is_pr = bool(discussion.get("isPullRequest", False))
|
| 776 |
+
author_name = self.extract_author_name(discussion)
|
| 777 |
+
repo_id = event.get("repoId", "unknown-repo")
|
| 778 |
+
discussion_num = discussion.get("num", "unknown-num")
|
| 779 |
+
event_id = event.get("eventId", "unknown-event")
|
| 780 |
|
| 781 |
+
if is_pr:
|
| 782 |
+
if author_name and author_name.lower() == self.username.lower():
|
| 783 |
+
return "pr_opened", f"{repo_id}::pr::{discussion_num}"
|
| 784 |
+
return "pr_activity", f"event::{event_id}"
|
| 785 |
|
| 786 |
+
if author_name and author_name.lower() == self.username.lower():
|
| 787 |
+
return "thread_opened", f"{repo_id}::thread::{discussion_num}"
|
| 788 |
+
return "discussion_activity", f"event::{event_id}"
|
| 789 |
|
| 790 |
+
def extract_business_timestamp(self, bucket_key: str, event: dict, discussion: dict):
|
| 791 |
+
event_ts = self.parse_ts(event.get("time"))
|
| 792 |
+
created_ts = self.parse_ts(discussion.get("createdAt"))
|
|
|
|
| 793 |
|
| 794 |
+
if bucket_key in {"pr_opened", "thread_opened"}:
|
| 795 |
+
return created_ts
|
|
|
|
|
|
|
| 796 |
|
| 797 |
+
if bucket_key in {"pr_activity", "discussion_activity"}:
|
| 798 |
+
if event_ts is None:
|
| 799 |
+
return None
|
| 800 |
+
if created_ts is not None:
|
| 801 |
+
delta_seconds = abs((event_ts - created_ts).total_seconds())
|
| 802 |
+
if delta_seconds <= self.OPEN_EVENT_EPSILON_SECONDS:
|
| 803 |
+
return None
|
| 804 |
+
return event_ts
|
| 805 |
|
| 806 |
+
return None
|
|
|
|
| 807 |
|
| 808 |
+
def process_event(
|
| 809 |
+
self,
|
| 810 |
+
*,
|
| 811 |
+
event: dict,
|
| 812 |
+
event_ts,
|
| 813 |
+
year_start,
|
| 814 |
+
year_end,
|
| 815 |
+
state: dict,
|
| 816 |
+
):
|
| 817 |
+
_ = event_ts
|
| 818 |
|
| 819 |
+
discussion = event.get("discussionData") or {}
|
| 820 |
+
if not isinstance(discussion, dict):
|
| 821 |
+
return
|
| 822 |
|
| 823 |
+
bucket_key, unique_id = self.classify_event(event, discussion)
|
| 824 |
+
if not bucket_key or not unique_id:
|
| 825 |
+
return
|
| 826 |
|
| 827 |
+
business_ts = self.extract_business_timestamp(bucket_key, event, discussion)
|
| 828 |
+
if business_ts is None:
|
| 829 |
+
return
|
| 830 |
|
| 831 |
+
if not (year_start <= business_ts <= year_end):
|
| 832 |
+
return
|
| 833 |
|
| 834 |
+
if unique_id in state["seen_ids"][bucket_key]:
|
| 835 |
+
return
|
| 836 |
|
| 837 |
+
state["seen_ids"][bucket_key].add(unique_id)
|
| 838 |
+
state["dates"][bucket_key].append(business_ts.tz_localize(None).date())
|
|
|
|
| 839 |
|
| 840 |
+
def build_bucket(self, key: str, dates: list, selected_year: int):
|
| 841 |
+
return {
|
| 842 |
+
"title": self.TITLE_MAP[key],
|
| 843 |
+
"fetched": True,
|
| 844 |
+
"dates": dates,
|
| 845 |
+
"count": len(dates),
|
| 846 |
+
"logs": f"Found {len(dates)} {self.LOG_LABEL_MAP[key]} in {selected_year}.",
|
| 847 |
+
}
|
| 848 |
|
| 849 |
+
def build_result(
|
| 850 |
+
self,
|
| 851 |
+
*,
|
| 852 |
+
username: str,
|
| 853 |
+
selected_year: int,
|
| 854 |
+
state: dict,
|
| 855 |
+
logs: list[str],
|
| 856 |
+
pages_fetched: int,
|
| 857 |
+
oldest_event_ts_seen,
|
| 858 |
+
seen_target_year: bool,
|
| 859 |
+
crossed_before_year_start: bool,
|
| 860 |
+
):
|
| 861 |
+
logs = list(logs)
|
| 862 |
+
logs.append(f"Fetched {pages_fetched} page(s) from recent-activity.")
|
| 863 |
+
if oldest_event_ts_seen is not None:
|
| 864 |
+
logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
|
| 865 |
+
logs.append(f"Seen target year: {seen_target_year}.")
|
| 866 |
+
logs.append(f"Crossed before year start: {crossed_before_year_start}.")
|
| 867 |
+
logs.append(
|
| 868 |
+
" | ".join(
|
| 869 |
+
f"{self.TITLE_MAP[key]}: {len(state['dates'][key])}"
|
| 870 |
+
for key in ["pr_opened", "pr_activity", "thread_opened", "discussion_activity"]
|
| 871 |
+
)
|
| 872 |
+
)
|
| 873 |
|
| 874 |
+
combined_logs = "\n\n".join(
|
| 875 |
+
[
|
| 876 |
+
"[Pull Requests]\n" + self.build_bucket("pr_opened", state["dates"]["pr_opened"], selected_year)["logs"],
|
| 877 |
+
"[PR Activity]\n" + self.build_bucket("pr_activity", state["dates"]["pr_activity"], selected_year)["logs"],
|
| 878 |
+
"[Community Threads]\n" + self.build_bucket("thread_opened", state["dates"]["thread_opened"], selected_year)["logs"],
|
| 879 |
+
"[Community Activity]\n" + self.build_bucket("discussion_activity", state["dates"]["discussion_activity"], selected_year)["logs"],
|
| 880 |
+
"[Fetch]\n" + "\n".join(logs),
|
| 881 |
+
]
|
| 882 |
+
)
|
| 883 |
|
| 884 |
+
return {
|
| 885 |
+
"username": username,
|
| 886 |
+
"selected_year": selected_year,
|
| 887 |
+
"fetched": True,
|
| 888 |
+
"pr_opened": self.build_bucket("pr_opened", state["dates"]["pr_opened"], selected_year),
|
| 889 |
+
"pr_activity": self.build_bucket("pr_activity", state["dates"]["pr_activity"], selected_year),
|
| 890 |
+
"thread_opened": self.build_bucket("thread_opened", state["dates"]["thread_opened"], selected_year),
|
| 891 |
+
"discussion_activity": self.build_bucket("discussion_activity", state["dates"]["discussion_activity"], selected_year),
|
| 892 |
+
"logs": combined_logs,
|
| 893 |
+
}
|
| 894 |
|
| 895 |
+
def build_error_result(
|
| 896 |
+
self,
|
| 897 |
+
*,
|
| 898 |
+
username: str,
|
| 899 |
+
selected_year: int,
|
| 900 |
+
error,
|
| 901 |
+
logs: list[str],
|
| 902 |
+
):
|
| 903 |
+
_ = logs
|
| 904 |
+
return {
|
| 905 |
"username": username,
|
| 906 |
"selected_year": selected_year,
|
| 907 |
"fetched": True,
|
| 908 |
+
"pr_opened": build_failed_bucket("Pull Requests"),
|
| 909 |
+
"pr_activity": build_failed_bucket("PR Activity"),
|
| 910 |
+
"thread_opened": build_failed_bucket("Community Threads"),
|
| 911 |
+
"discussion_activity": build_failed_bucket("Community Activity"),
|
| 912 |
+
"logs": f"Community fetch failed: {str(error)}",
|
| 913 |
}
|
| 914 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 915 |
|
| 916 |
+
class BucketedActivityProcessor:
|
| 917 |
+
def __init__(
|
| 918 |
+
self,
|
| 919 |
+
*,
|
| 920 |
+
bucket_titles: dict[str, str],
|
| 921 |
+
bucket_log_labels: dict[str, str],
|
| 922 |
+
classifier,
|
| 923 |
+
error_label: str,
|
| 924 |
+
):
|
| 925 |
+
self.bucket_titles = bucket_titles
|
| 926 |
+
self.bucket_log_labels = bucket_log_labels
|
| 927 |
+
self.classifier = classifier
|
| 928 |
+
self.error_label = error_label
|
| 929 |
+
|
| 930 |
+
def init_state(self):
|
| 931 |
+
return {
|
| 932 |
+
"bucket_dates": {key: [] for key in self.bucket_titles},
|
| 933 |
+
"bucket_seen_ids": {key: set() for key in self.bucket_titles},
|
| 934 |
+
}
|
| 935 |
+
|
| 936 |
+
def process_event(
|
| 937 |
+
self,
|
| 938 |
+
*,
|
| 939 |
+
event: dict,
|
| 940 |
+
event_ts,
|
| 941 |
+
year_start,
|
| 942 |
+
year_end,
|
| 943 |
+
state: dict,
|
| 944 |
+
):
|
| 945 |
+
if event_ts is None:
|
| 946 |
+
return
|
| 947 |
+
|
| 948 |
+
if not (year_start <= event_ts <= year_end):
|
| 949 |
+
return
|
| 950 |
+
|
| 951 |
+
bucket_key, unique_id = self.classifier(event)
|
| 952 |
+
|
| 953 |
+
if not bucket_key or not unique_id:
|
| 954 |
+
return
|
| 955 |
+
|
| 956 |
+
if bucket_key not in state["bucket_dates"]:
|
| 957 |
+
return
|
| 958 |
+
|
| 959 |
+
if unique_id in state["bucket_seen_ids"][bucket_key]:
|
| 960 |
+
return
|
| 961 |
+
|
| 962 |
+
state["bucket_seen_ids"][bucket_key].add(unique_id)
|
| 963 |
+
state["bucket_dates"][bucket_key].append(event_ts.tz_localize(None).date())
|
| 964 |
+
|
| 965 |
+
def build_result(
|
| 966 |
+
self,
|
| 967 |
+
*,
|
| 968 |
+
username: str,
|
| 969 |
+
selected_year: int,
|
| 970 |
+
state: dict,
|
| 971 |
+
logs: list[str],
|
| 972 |
+
pages_fetched: int,
|
| 973 |
+
oldest_event_ts_seen,
|
| 974 |
+
seen_target_year: bool,
|
| 975 |
+
crossed_before_year_start: bool,
|
| 976 |
+
):
|
| 977 |
+
logs = list(logs)
|
| 978 |
logs.append(f"Fetched {pages_fetched} page(s) from recent-activity.")
|
| 979 |
if oldest_event_ts_seen is not None:
|
| 980 |
logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
|
|
|
|
| 982 |
logs.append(f"Crossed before year start: {crossed_before_year_start}.")
|
| 983 |
logs.append(
|
| 984 |
" | ".join(
|
| 985 |
+
f"{self.bucket_titles[key]}: {len(state['bucket_dates'][key])}"
|
| 986 |
+
for key in self.bucket_titles
|
| 987 |
)
|
| 988 |
)
|
| 989 |
|
| 990 |
+
result = {
|
| 991 |
+
"username": username,
|
| 992 |
+
"selected_year": selected_year,
|
| 993 |
+
"fetched": True,
|
| 994 |
+
"logs": "\n".join(logs),
|
| 995 |
+
}
|
| 996 |
+
|
| 997 |
+
for key, title in self.bucket_titles.items():
|
| 998 |
+
dates = state["bucket_dates"][key]
|
| 999 |
+
result[key] = build_fetched_bucket(
|
| 1000 |
+
title=title,
|
| 1001 |
+
dates=dates,
|
| 1002 |
+
selected_year=selected_year,
|
| 1003 |
+
label=self.bucket_log_labels[key],
|
| 1004 |
+
)
|
| 1005 |
|
|
|
|
| 1006 |
return result
|
| 1007 |
|
| 1008 |
+
def build_error_result(
|
| 1009 |
+
self,
|
| 1010 |
+
*,
|
| 1011 |
+
username: str,
|
| 1012 |
+
selected_year: int,
|
| 1013 |
+
error,
|
| 1014 |
+
logs: list[str],
|
| 1015 |
+
):
|
| 1016 |
+
_ = logs
|
| 1017 |
+
result = {
|
| 1018 |
"username": username,
|
| 1019 |
"selected_year": selected_year,
|
| 1020 |
"fetched": True,
|
| 1021 |
+
"logs": f"{self.error_label} fetch failed: {str(error)}",
|
| 1022 |
}
|
| 1023 |
|
| 1024 |
+
for key, title in self.bucket_titles.items():
|
| 1025 |
+
result[key] = build_failed_bucket(title)
|
| 1026 |
|
| 1027 |
+
return result
|
| 1028 |
|
| 1029 |
|
| 1030 |
+
# ============================================================
|
| 1031 |
+
# Activity classifiers + config
|
| 1032 |
+
# ============================================================
|
| 1033 |
def classify_upvote_event(event: dict):
|
| 1034 |
target_type = event.get("targetType")
|
| 1035 |
target = event.get("target") or {}
|
|
|
|
| 1180 |
)
|
| 1181 |
|
| 1182 |
|
| 1183 |
+
# ============================================================
|
| 1184 |
+
# Public activity fetch functions
|
| 1185 |
+
# ============================================================
|
| 1186 |
+
def fetch_all_community_activity(username: str, selected_year: int):
|
| 1187 |
+
username = (username or "").strip()
|
| 1188 |
+
selected_year = int(selected_year)
|
| 1189 |
+
|
| 1190 |
+
cache_key = (username.lower(), selected_year, "community")
|
| 1191 |
+
cached = get_cache(cache_key)
|
| 1192 |
+
if cached:
|
| 1193 |
+
return cached
|
| 1194 |
+
|
| 1195 |
+
if not username:
|
| 1196 |
+
state = empty_community_state(selected_year)
|
| 1197 |
+
state["fetched"] = True
|
| 1198 |
+
state["logs"] = "No username provided."
|
| 1199 |
+
set_cache(cache_key, state)
|
| 1200 |
+
return state
|
| 1201 |
+
|
| 1202 |
+
processor = CommunityActivityProcessor(username=username)
|
| 1203 |
+
|
| 1204 |
+
result = collect_recent_activity(
|
| 1205 |
username=username,
|
| 1206 |
selected_year=selected_year,
|
| 1207 |
+
activity_type="discussion",
|
| 1208 |
+
processor=processor,
|
| 1209 |
+
build_session_fn=build_session,
|
| 1210 |
+
rate_limiter_obj=recent_activity_rate_limiter,
|
| 1211 |
+
)
|
| 1212 |
+
|
| 1213 |
+
set_cache(cache_key, result)
|
| 1214 |
+
return result
|
| 1215 |
+
|
| 1216 |
+
|
| 1217 |
+
def fetch_single_section_activity(section_key: str, username: str, selected_year: int):
|
| 1218 |
+
username = (username or "").strip()
|
| 1219 |
+
selected_year = int(selected_year)
|
| 1220 |
+
cfg = SINGLE_SECTION_CONFIG[section_key]
|
| 1221 |
+
|
| 1222 |
+
cache_key = (username.lower(), selected_year, cfg["cache_scope"])
|
| 1223 |
+
cached = get_cache(cache_key)
|
| 1224 |
+
if cached:
|
| 1225 |
+
return cached
|
| 1226 |
+
|
| 1227 |
+
if not username:
|
| 1228 |
+
state = empty_single_section_state(section_key, selected_year)
|
| 1229 |
+
state["fetched"] = True
|
| 1230 |
+
state["logs"] = "No username provided."
|
| 1231 |
+
set_cache(cache_key, state)
|
| 1232 |
+
return state
|
| 1233 |
+
|
| 1234 |
+
processor = BucketedActivityProcessor(
|
| 1235 |
bucket_titles=cfg["bucket_titles"],
|
| 1236 |
bucket_log_labels=cfg["bucket_log_labels"],
|
|
|
|
| 1237 |
classifier=cfg["classifier"],
|
| 1238 |
+
error_label=cfg["cache_scope"].capitalize(),
|
| 1239 |
)
|
| 1240 |
|
| 1241 |
+
result = collect_recent_activity(
|
| 1242 |
+
username=username,
|
| 1243 |
+
selected_year=selected_year,
|
| 1244 |
+
activity_type=cfg["activity_type"],
|
| 1245 |
+
processor=processor,
|
| 1246 |
+
build_session_fn=build_session,
|
| 1247 |
+
rate_limiter_obj=recent_activity_rate_limiter,
|
| 1248 |
+
)
|
| 1249 |
+
|
| 1250 |
+
set_cache(cache_key, result)
|
| 1251 |
+
return result
|
| 1252 |
+
|
| 1253 |
|
| 1254 |
# ============================================================
|
| 1255 |
# Contribution grid helpers
|