fffiloni commited on
Commit
f2d6dd9
·
verified ·
1 Parent(s): cc71c22

refactor: unify recent-activity logic with generic engine + processors

Browse files
Files changed (1) hide show
  1. app.py +465 -385
app.py CHANGED
@@ -58,6 +58,9 @@ def build_session(accept: str) -> requests.Session:
58
  return session
59
 
60
 
 
 
 
61
  def normalize_recent_activity_payload(payload):
62
  if isinstance(payload, list):
63
  return payload, None
@@ -76,15 +79,9 @@ def normalize_recent_activity_payload(payload):
76
 
77
 
78
  def get_year_bounds(selected_year: int):
79
- start_ts = pd.Timestamp(f"{selected_year}-01-01", tz="UTC")
80
- end_ts = pd.Timestamp(f"{selected_year}-12-31 23:59:59.999999", tz="UTC")
81
- return start_ts, end_ts
82
-
83
-
84
- def should_stop_after_crossing_year_start(
85
- crossed_before_year_start: bool,
86
- ) -> bool:
87
- return crossed_before_year_start
88
 
89
 
90
  def build_events_page_signature(events: list[dict]) -> tuple:
@@ -104,6 +101,71 @@ def build_events_page_signature(events: list[dict]) -> tuple:
104
  return tuple(signature)
105
 
106
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
107
  # ============================================================
108
  # Thread-safe rate limiters
109
  # ============================================================
@@ -485,7 +547,7 @@ def build_invalid_section_state(username: str, selected_year: int, empty_state_f
485
 
486
 
487
  # ============================================================
488
- # Community activity fetch
489
  # ============================================================
490
  def empty_community_state(selected_year: int):
491
  return {
@@ -500,36 +562,30 @@ def empty_community_state(selected_year: int):
500
  }
501
 
502
 
503
- def fetch_user_discussion_activity(
 
 
 
 
504
  username: str,
505
  selected_year: int,
506
- mode: str,
 
 
 
 
 
 
507
  ):
508
  username = (username or "").strip()
509
  selected_year = int(selected_year)
510
 
511
- title_map = {
512
- "pr_opened": "Pull Requests",
513
- "pr_activity": "PR Activity",
514
- "thread_opened": "Community Threads",
515
- "discussion_activity": "Community Activity",
516
- }
517
- title = title_map.get(mode, "Community")
518
-
519
- if not username:
520
- return {
521
- "title": title,
522
- "fetched": True,
523
- "dates": [],
524
- "count": 0,
525
- "logs": "No username provided.",
526
- }
527
 
528
- dates = []
529
- seen_ids = set()
530
- skip = 0
531
- limit = RECENT_ACTIVITY_LIMIT
532
  cursor = None
 
533
  pages_fetched = 0
534
 
535
  seen_target_year = False
@@ -540,93 +596,19 @@ def fetch_user_discussion_activity(
540
  seen_cursors = set()
541
  seen_page_signatures = set()
542
 
543
- OPEN_EVENT_EPSILON_SECONDS = 90
544
  logs = []
545
 
546
- year_start, year_end = get_year_bounds(selected_year)
547
- session = build_session("application/json")
548
-
549
- def extract_author_name(discussion: dict) -> str | None:
550
- author = discussion.get("author")
551
- if isinstance(author, str):
552
- return author
553
- if isinstance(author, dict):
554
- for key in ["name", "username", "login"]:
555
- value = author.get(key)
556
- if isinstance(value, str):
557
- return value
558
- return None
559
-
560
- def parse_ts(raw_value):
561
- if not raw_value:
562
- return None
563
- try:
564
- return pd.to_datetime(raw_value, utc=True)
565
- except Exception:
566
- return None
567
-
568
- def event_matches_mode(event: dict, discussion: dict) -> tuple[bool, str | None]:
569
- is_pr = bool(discussion.get("isPullRequest", False))
570
- author_name = extract_author_name(discussion)
571
- repo_id = event.get("repoId", "unknown-repo")
572
- discussion_num = discussion.get("num", "unknown-num")
573
- event_id = event.get("eventId", "unknown-event")
574
-
575
- if mode == "pr_opened":
576
- if not is_pr:
577
- return False, None
578
- if not author_name or author_name.lower() != username.lower():
579
- return False, None
580
- return True, f"{repo_id}::pr::{discussion_num}"
581
-
582
- if mode == "pr_activity":
583
- if not is_pr:
584
- return False, None
585
- return True, f"event::{event_id}"
586
-
587
- if mode == "thread_opened":
588
- if is_pr:
589
- return False, None
590
- if not author_name or author_name.lower() != username.lower():
591
- return False, None
592
- return True, f"{repo_id}::thread::{discussion_num}"
593
-
594
- if mode == "discussion_activity":
595
- if is_pr:
596
- return False, None
597
- return True, f"event::{event_id}"
598
-
599
- return False, None
600
-
601
- def extract_timestamp_for_mode(event: dict, discussion: dict):
602
- event_ts = parse_ts(event.get("time"))
603
- created_ts = parse_ts(discussion.get("createdAt"))
604
-
605
- if mode in {"pr_opened", "thread_opened"}:
606
- return created_ts
607
-
608
- if mode in {"pr_activity", "discussion_activity"}:
609
- if event_ts is None:
610
- return None
611
- if created_ts is not None:
612
- delta_seconds = abs((event_ts - created_ts).total_seconds())
613
- if delta_seconds <= OPEN_EVENT_EPSILON_SECONDS:
614
- return None
615
- return event_ts
616
-
617
- return None
618
-
619
  try:
620
  while True:
621
- if pages_fetched >= RECENT_ACTIVITY_MAX_PAGES:
622
  logs.append("Stopped due to safety max pages.")
623
  break
624
 
625
- recent_activity_rate_limiter.wait()
626
 
627
  params = {
628
  "limit": limit,
629
- "activityType": "discussion",
630
  "feedType": "user",
631
  "entity": username,
632
  }
@@ -669,313 +651,330 @@ def fetch_user_discussion_activity(
669
  if not isinstance(event, dict):
670
  continue
671
 
672
- event_ts = parse_ts(event.get("time"))
673
- if event_ts is not None:
674
- if oldest_event_ts_seen is None or event_ts < oldest_event_ts_seen:
675
- oldest_event_ts_seen = event_ts
676
- if page_oldest_event_ts is None or event_ts < page_oldest_event_ts:
677
- page_oldest_event_ts = event_ts
678
-
679
- if year_start <= event_ts <= year_end:
680
- seen_target_year = True
681
- elif event_ts < year_start:
682
- crossed_before_year_start = True
683
-
684
- discussion = event.get("discussionData") or {}
685
- if not isinstance(discussion, dict):
686
- continue
687
-
688
- keep, unique_id = event_matches_mode(event, discussion)
689
- if not keep or not unique_id:
690
- continue
691
-
692
- dt = extract_timestamp_for_mode(event, discussion)
693
- if dt is None:
694
- continue
695
-
696
- if year_start <= dt <= year_end:
697
- seen_target_year = True
698
- elif dt < year_start:
699
- continue
700
- elif dt > year_end:
701
- continue
702
-
703
- if unique_id in seen_ids:
704
- continue
705
-
706
- seen_ids.add(unique_id)
707
- dates.append(dt.tz_localize(None).date())
708
-
709
- if should_stop_after_crossing_year_start(crossed_before_year_start):
710
  logs.append("Stopped after crossing before selected year.")
711
  break
712
 
713
- if previous_oldest_event_ts_seen is not None and page_oldest_event_ts is not None:
714
- if page_oldest_event_ts >= previous_oldest_event_ts_seen:
715
- stalled_oldest_count += 1
716
- else:
717
- stalled_oldest_count = 0
718
 
719
- if stalled_oldest_count >= RECENT_ACTIVITY_MAX_STALLED_PAGES:
720
  logs.append("Stopped because pagination no longer progressed toward older events.")
721
  break
722
 
723
- if page_oldest_event_ts is not None:
724
- previous_oldest_event_ts_seen = page_oldest_event_ts
725
-
726
  if next_cursor:
727
  cursor = next_cursor
728
  else:
729
  skip += limit
730
 
731
- logs.append(f"Fetched {pages_fetched} page(s) from recent-activity.")
732
- if oldest_event_ts_seen is not None:
733
- logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
734
- logs.append(f"Seen target year: {seen_target_year}.")
735
- logs.append(f"Crossed before year start: {crossed_before_year_start}.")
736
- logs.append(f"Found {len(dates)} unique {mode} event(s) in {selected_year}.")
737
-
738
- return {
739
- "title": title,
740
- "fetched": True,
741
- "dates": dates,
742
- "count": len(dates),
743
- "logs": "\n".join(logs),
744
- }
745
 
746
  except Exception as e:
747
- return {
748
- "title": title,
749
- "fetched": True,
750
- "dates": [],
751
- "count": 0,
752
- "logs": f"{title} fetch failed: {str(e)}",
753
- }
754
-
755
-
756
- def fetch_all_community_activity(username: str, selected_year: int):
757
- username = (username or "").strip()
758
- selected_year = int(selected_year)
759
-
760
- cache_key = (username.lower(), selected_year, "community")
761
- cached = get_cache(cache_key)
762
- if cached:
763
- return cached
764
-
765
- if not username:
766
- state = empty_community_state(selected_year)
767
- state["fetched"] = True
768
- state["logs"] = "No username provided."
769
- set_cache(cache_key, state)
770
- return state
771
-
772
- pr_opened = fetch_user_discussion_activity(username, selected_year, "pr_opened")
773
- pr_activity = fetch_user_discussion_activity(username, selected_year, "pr_activity")
774
- thread_opened = fetch_user_discussion_activity(username, selected_year, "thread_opened")
775
- discussion_activity = fetch_user_discussion_activity(username, selected_year, "discussion_activity")
776
-
777
- combined_logs = "\n\n".join(
778
- [
779
- "[Pull Requests]\n" + pr_opened["logs"],
780
- "[PR Activity]\n" + pr_activity["logs"],
781
- "[Community Threads]\n" + thread_opened["logs"],
782
- "[Community Activity]\n" + discussion_activity["logs"],
783
- ]
784
- )
785
-
786
- result = {
787
- "username": username,
788
- "selected_year": selected_year,
789
- "fetched": True,
790
- "pr_opened": pr_opened,
791
- "pr_activity": pr_activity,
792
- "thread_opened": thread_opened,
793
- "discussion_activity": discussion_activity,
794
- "logs": combined_logs,
795
- }
796
-
797
- set_cache(cache_key, result)
798
- return result
799
 
800
 
801
  # ============================================================
802
- # Bucketed recent-activity fetch
803
  # ============================================================
804
- def fetch_bucketed_recent_activity(
805
- username: str,
806
- selected_year: int,
807
- cache_scope: str,
808
- activity_type: str,
809
- bucket_titles: dict[str, str],
810
- bucket_log_labels: dict[str, str],
811
- empty_logs_text: str,
812
- classifier,
813
- ):
814
- username = (username or "").strip()
815
- selected_year = int(selected_year)
816
-
817
- cache_key = (username.lower(), selected_year, cache_scope)
818
- cached = get_cache(cache_key)
819
- if cached:
820
- return cached
821
-
822
- if not username:
823
- state = empty_bucketed_activity_state(selected_year, empty_logs_text, bucket_titles)
824
- state["fetched"] = True
825
- state["logs"] = "No username provided."
826
- set_cache(cache_key, state)
827
- return state
828
-
829
- bucket_dates = {key: [] for key in bucket_titles}
830
- bucket_seen_ids = {key: set() for key in bucket_titles}
831
-
832
- cursor = None
833
- skip = 0
834
- limit = RECENT_ACTIVITY_LIMIT
835
- pages_fetched = 0
836
-
837
- seen_target_year = False
838
- crossed_before_year_start = False
839
- oldest_event_ts_seen = None
840
- previous_oldest_event_ts_seen = None
841
- stalled_oldest_count = 0
842
- seen_cursors = set()
843
- seen_page_signatures = set()
844
-
845
- logs = []
846
-
847
- year_start, year_end = get_year_bounds(selected_year)
848
- session = build_session("application/json")
849
-
850
- try:
851
- while True:
852
- if pages_fetched >= RECENT_ACTIVITY_MAX_PAGES:
853
- logs.append("Stopped due to safety max pages.")
854
- break
855
-
856
- recent_activity_rate_limiter.wait()
857
-
858
- params = {
859
- "limit": limit,
860
- "activityType": activity_type,
861
- "feedType": "user",
862
- "entity": username,
863
- }
864
- if cursor:
865
- params["cursor"] = cursor
866
- else:
867
- params["skip"] = skip
868
-
869
- response = session.get(
870
- "https://huggingface.co/api/recent-activity",
871
- params=params,
872
- timeout=20,
873
- )
874
- response.raise_for_status()
875
-
876
- payload = response.json()
877
- pages_fetched += 1
878
 
879
- events, next_cursor = normalize_recent_activity_payload(payload)
 
880
 
881
- if not events:
882
- logs.append("No more events.")
883
- break
 
 
 
 
 
 
 
 
 
 
 
 
884
 
885
- page_signature = build_events_page_signature(events)
886
- if page_signature in seen_page_signatures:
887
- logs.append("Stopped because recent-activity returned a repeated page.")
888
- break
889
- seen_page_signatures.add(page_signature)
 
 
 
 
 
890
 
891
- if next_cursor:
892
- if next_cursor in seen_cursors:
893
- logs.append("Stopped because recent-activity returned a repeated cursor.")
894
- break
895
- seen_cursors.add(next_cursor)
 
 
896
 
897
- page_oldest_event_ts = None
 
 
 
 
 
898
 
899
- for event in events:
900
- if not isinstance(event, dict):
901
- continue
 
902
 
903
- raw_time = event.get("time")
904
- if not raw_time:
905
- continue
906
 
907
- try:
908
- event_ts = pd.to_datetime(raw_time, utc=True)
909
- except Exception:
910
- continue
911
 
912
- if oldest_event_ts_seen is None or event_ts < oldest_event_ts_seen:
913
- oldest_event_ts_seen = event_ts
914
- if page_oldest_event_ts is None or event_ts < page_oldest_event_ts:
915
- page_oldest_event_ts = event_ts
916
 
917
- if year_start <= event_ts <= year_end:
918
- seen_target_year = True
919
- elif event_ts < year_start:
920
- crossed_before_year_start = True
 
 
 
 
921
 
922
- if event_ts > year_end:
923
- continue
924
 
925
- if event_ts < year_start:
926
- continue
 
 
 
 
 
 
 
 
927
 
928
- bucket_key, unique_id = classifier(event)
 
 
929
 
930
- if not bucket_key or not unique_id:
931
- continue
 
932
 
933
- if bucket_key not in bucket_dates:
934
- continue
 
935
 
936
- if unique_id in bucket_seen_ids[bucket_key]:
937
- continue
938
 
939
- bucket_seen_ids[bucket_key].add(unique_id)
940
- bucket_dates[bucket_key].append(event_ts.tz_localize(None).date())
941
 
942
- if should_stop_after_crossing_year_start(crossed_before_year_start):
943
- logs.append("Stopped after crossing before selected year.")
944
- break
945
 
946
- if previous_oldest_event_ts_seen is not None and page_oldest_event_ts is not None:
947
- if page_oldest_event_ts >= previous_oldest_event_ts_seen:
948
- stalled_oldest_count += 1
949
- else:
950
- stalled_oldest_count = 0
 
 
 
951
 
952
- if stalled_oldest_count >= RECENT_ACTIVITY_MAX_STALLED_PAGES:
953
- logs.append("Stopped because pagination no longer progressed toward older events.")
954
- break
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
955
 
956
- if page_oldest_event_ts is not None:
957
- previous_oldest_event_ts_seen = page_oldest_event_ts
 
 
 
 
 
 
 
958
 
959
- if next_cursor:
960
- cursor = next_cursor
961
- else:
962
- skip += limit
 
 
 
 
 
 
963
 
964
- result = {
 
 
 
 
 
 
 
 
 
965
  "username": username,
966
  "selected_year": selected_year,
967
  "fetched": True,
968
- "logs": "",
 
 
 
 
969
  }
970
 
971
- for key, title in bucket_titles.items():
972
- result[key] = build_fetched_bucket(
973
- title=title,
974
- dates=bucket_dates[key],
975
- selected_year=selected_year,
976
- label=bucket_log_labels[key],
977
- )
978
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
979
  logs.append(f"Fetched {pages_fetched} page(s) from recent-activity.")
980
  if oldest_event_ts_seen is not None:
981
  logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
@@ -983,30 +982,54 @@ def fetch_bucketed_recent_activity(
983
  logs.append(f"Crossed before year start: {crossed_before_year_start}.")
984
  logs.append(
985
  " | ".join(
986
- f"{bucket_titles[key]}: {len(bucket_dates[key])}"
987
- for key in bucket_titles
988
  )
989
  )
990
 
991
- result["logs"] = "\n".join(logs)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
992
 
993
- set_cache(cache_key, result)
994
  return result
995
 
996
- except Exception as e:
997
- failed = {
 
 
 
 
 
 
 
 
998
  "username": username,
999
  "selected_year": selected_year,
1000
  "fetched": True,
1001
- "logs": f"{cache_scope.capitalize()} fetch failed: {str(e)}",
1002
  }
1003
 
1004
- for key, title in bucket_titles.items():
1005
- failed[key] = build_failed_bucket(title)
1006
 
1007
- return failed
1008
 
1009
 
 
 
 
1010
  def classify_upvote_event(event: dict):
1011
  target_type = event.get("targetType")
1012
  target = event.get("target") or {}
@@ -1157,19 +1180,76 @@ def empty_single_section_state(section_key: str, selected_year: int):
1157
  )
1158
 
1159
 
1160
- def fetch_single_section_activity(section_key: str, username: str, selected_year: int):
1161
- cfg = SINGLE_SECTION_CONFIG[section_key]
1162
- return fetch_bucketed_recent_activity(
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1163
  username=username,
1164
  selected_year=selected_year,
1165
- cache_scope=cfg["cache_scope"],
1166
- activity_type=cfg["activity_type"],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1167
  bucket_titles=cfg["bucket_titles"],
1168
  bucket_log_labels=cfg["bucket_log_labels"],
1169
- empty_logs_text=cfg["empty_logs_text"],
1170
  classifier=cfg["classifier"],
 
1171
  )
1172
 
 
 
 
 
 
 
 
 
 
 
 
 
1173
 
1174
  # ============================================================
1175
  # Contribution grid helpers
 
58
  return session
59
 
60
 
61
+ # ============================================================
62
+ # Recent-activity shared helpers
63
+ # ============================================================
64
  def normalize_recent_activity_payload(payload):
65
  if isinstance(payload, list):
66
  return payload, None
 
79
 
80
 
81
  def get_year_bounds(selected_year: int):
82
+ year_start = pd.Timestamp(f"{selected_year}-01-01", tz="UTC")
83
+ year_end = pd.Timestamp(f"{selected_year}-12-31 23:59:59.999999", tz="UTC")
84
+ return year_start, year_end
 
 
 
 
 
 
85
 
86
 
87
  def build_events_page_signature(events: list[dict]) -> tuple:
 
101
  return tuple(signature)
102
 
103
 
104
+ def parse_recent_activity_event_ts(event: dict):
105
+ raw_time = event.get("time")
106
+ if not raw_time:
107
+ return None
108
+ try:
109
+ return pd.to_datetime(raw_time, utc=True)
110
+ except Exception:
111
+ return None
112
+
113
+
114
+ def update_recent_activity_progress(
115
+ event_ts,
116
+ *,
117
+ year_start,
118
+ year_end,
119
+ oldest_event_ts_seen,
120
+ page_oldest_event_ts,
121
+ seen_target_year,
122
+ crossed_before_year_start,
123
+ ):
124
+ if event_ts is None:
125
+ return (
126
+ oldest_event_ts_seen,
127
+ page_oldest_event_ts,
128
+ seen_target_year,
129
+ crossed_before_year_start,
130
+ )
131
+
132
+ if oldest_event_ts_seen is None or event_ts < oldest_event_ts_seen:
133
+ oldest_event_ts_seen = event_ts
134
+
135
+ if page_oldest_event_ts is None or event_ts < page_oldest_event_ts:
136
+ page_oldest_event_ts = event_ts
137
+
138
+ if year_start <= event_ts <= year_end:
139
+ seen_target_year = True
140
+ elif event_ts < year_start:
141
+ crossed_before_year_start = True
142
+
143
+ return (
144
+ oldest_event_ts_seen,
145
+ page_oldest_event_ts,
146
+ seen_target_year,
147
+ crossed_before_year_start,
148
+ )
149
+
150
+
151
+ def update_stalled_pagination_state(
152
+ *,
153
+ page_oldest_event_ts,
154
+ previous_oldest_event_ts_seen,
155
+ stalled_oldest_count,
156
+ ):
157
+ if previous_oldest_event_ts_seen is not None and page_oldest_event_ts is not None:
158
+ if page_oldest_event_ts >= previous_oldest_event_ts_seen:
159
+ stalled_oldest_count += 1
160
+ else:
161
+ stalled_oldest_count = 0
162
+
163
+ if page_oldest_event_ts is not None:
164
+ previous_oldest_event_ts_seen = page_oldest_event_ts
165
+
166
+ return previous_oldest_event_ts_seen, stalled_oldest_count
167
+
168
+
169
  # ============================================================
170
  # Thread-safe rate limiters
171
  # ============================================================
 
547
 
548
 
549
  # ============================================================
550
+ # Community activity state
551
  # ============================================================
552
  def empty_community_state(selected_year: int):
553
  return {
 
562
  }
563
 
564
 
565
+ # ============================================================
566
+ # Generic recent-activity engine
567
+ # ============================================================
568
+ def collect_recent_activity(
569
+ *,
570
  username: str,
571
  selected_year: int,
572
+ activity_type: str,
573
+ processor,
574
+ build_session_fn,
575
+ rate_limiter_obj,
576
+ limit: int = RECENT_ACTIVITY_LIMIT,
577
+ max_pages: int = RECENT_ACTIVITY_MAX_PAGES,
578
+ max_stalled_pages: int = RECENT_ACTIVITY_MAX_STALLED_PAGES,
579
  ):
580
  username = (username or "").strip()
581
  selected_year = int(selected_year)
582
 
583
+ processor_state = processor.init_state()
584
+ year_start, year_end = get_year_bounds(selected_year)
585
+ session = build_session_fn("application/json")
 
 
 
 
 
 
 
 
 
 
 
 
 
586
 
 
 
 
 
587
  cursor = None
588
+ skip = 0
589
  pages_fetched = 0
590
 
591
  seen_target_year = False
 
596
  seen_cursors = set()
597
  seen_page_signatures = set()
598
 
 
599
  logs = []
600
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
601
  try:
602
  while True:
603
+ if pages_fetched >= max_pages:
604
  logs.append("Stopped due to safety max pages.")
605
  break
606
 
607
+ rate_limiter_obj.wait()
608
 
609
  params = {
610
  "limit": limit,
611
+ "activityType": activity_type,
612
  "feedType": "user",
613
  "entity": username,
614
  }
 
651
  if not isinstance(event, dict):
652
  continue
653
 
654
+ event_ts = parse_recent_activity_event_ts(event)
655
+
656
+ (
657
+ oldest_event_ts_seen,
658
+ page_oldest_event_ts,
659
+ seen_target_year,
660
+ crossed_before_year_start,
661
+ ) = update_recent_activity_progress(
662
+ event_ts,
663
+ year_start=year_start,
664
+ year_end=year_end,
665
+ oldest_event_ts_seen=oldest_event_ts_seen,
666
+ page_oldest_event_ts=page_oldest_event_ts,
667
+ seen_target_year=seen_target_year,
668
+ crossed_before_year_start=crossed_before_year_start,
669
+ )
670
+
671
+ processor.process_event(
672
+ event=event,
673
+ event_ts=event_ts,
674
+ year_start=year_start,
675
+ year_end=year_end,
676
+ state=processor_state,
677
+ )
678
+
679
+ if crossed_before_year_start:
 
 
 
 
 
 
 
 
 
 
 
 
680
  logs.append("Stopped after crossing before selected year.")
681
  break
682
 
683
+ previous_oldest_event_ts_seen, stalled_oldest_count = update_stalled_pagination_state(
684
+ page_oldest_event_ts=page_oldest_event_ts,
685
+ previous_oldest_event_ts_seen=previous_oldest_event_ts_seen,
686
+ stalled_oldest_count=stalled_oldest_count,
687
+ )
688
 
689
+ if stalled_oldest_count >= max_stalled_pages:
690
  logs.append("Stopped because pagination no longer progressed toward older events.")
691
  break
692
 
 
 
 
693
  if next_cursor:
694
  cursor = next_cursor
695
  else:
696
  skip += limit
697
 
698
+ return processor.build_result(
699
+ username=username,
700
+ selected_year=selected_year,
701
+ state=processor_state,
702
+ logs=logs,
703
+ pages_fetched=pages_fetched,
704
+ oldest_event_ts_seen=oldest_event_ts_seen,
705
+ seen_target_year=seen_target_year,
706
+ crossed_before_year_start=crossed_before_year_start,
707
+ )
 
 
 
 
708
 
709
  except Exception as e:
710
+ return processor.build_error_result(
711
+ username=username,
712
+ selected_year=selected_year,
713
+ error=e,
714
+ logs=logs,
715
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
716
 
717
 
718
  # ============================================================
719
+ # Recent-activity processors
720
  # ============================================================
721
+ class CommunityActivityProcessor:
722
+ TITLE_MAP = {
723
+ "pr_opened": "Pull Requests",
724
+ "pr_activity": "PR Activity",
725
+ "thread_opened": "Community Threads",
726
+ "discussion_activity": "Community Activity",
727
+ }
728
+ LOG_LABEL_MAP = {
729
+ "pr_opened": "unique pr_opened event(s)",
730
+ "pr_activity": "unique pr_activity event(s)",
731
+ "thread_opened": "unique thread_opened event(s)",
732
+ "discussion_activity": "unique discussion_activity event(s)",
733
+ }
734
+ OPEN_EVENT_EPSILON_SECONDS = 90
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
735
 
736
+ def __init__(self, username: str):
737
+ self.username = (username or "").strip()
738
 
739
+ def init_state(self):
740
+ return {
741
+ "dates": {
742
+ "pr_opened": [],
743
+ "pr_activity": [],
744
+ "thread_opened": [],
745
+ "discussion_activity": [],
746
+ },
747
+ "seen_ids": {
748
+ "pr_opened": set(),
749
+ "pr_activity": set(),
750
+ "thread_opened": set(),
751
+ "discussion_activity": set(),
752
+ },
753
+ }
754
 
755
+ def extract_author_name(self, discussion: dict) -> str | None:
756
+ author = discussion.get("author")
757
+ if isinstance(author, str):
758
+ return author
759
+ if isinstance(author, dict):
760
+ for key in ["name", "username", "login"]:
761
+ value = author.get(key)
762
+ if isinstance(value, str):
763
+ return value
764
+ return None
765
 
766
+ def parse_ts(self, raw_value):
767
+ if not raw_value:
768
+ return None
769
+ try:
770
+ return pd.to_datetime(raw_value, utc=True)
771
+ except Exception:
772
+ return None
773
 
774
+ def classify_event(self, event: dict, discussion: dict) -> tuple[str | None, str | None]:
775
+ is_pr = bool(discussion.get("isPullRequest", False))
776
+ author_name = self.extract_author_name(discussion)
777
+ repo_id = event.get("repoId", "unknown-repo")
778
+ discussion_num = discussion.get("num", "unknown-num")
779
+ event_id = event.get("eventId", "unknown-event")
780
 
781
+ if is_pr:
782
+ if author_name and author_name.lower() == self.username.lower():
783
+ return "pr_opened", f"{repo_id}::pr::{discussion_num}"
784
+ return "pr_activity", f"event::{event_id}"
785
 
786
+ if author_name and author_name.lower() == self.username.lower():
787
+ return "thread_opened", f"{repo_id}::thread::{discussion_num}"
788
+ return "discussion_activity", f"event::{event_id}"
789
 
790
+ def extract_business_timestamp(self, bucket_key: str, event: dict, discussion: dict):
791
+ event_ts = self.parse_ts(event.get("time"))
792
+ created_ts = self.parse_ts(discussion.get("createdAt"))
 
793
 
794
+ if bucket_key in {"pr_opened", "thread_opened"}:
795
+ return created_ts
 
 
796
 
797
+ if bucket_key in {"pr_activity", "discussion_activity"}:
798
+ if event_ts is None:
799
+ return None
800
+ if created_ts is not None:
801
+ delta_seconds = abs((event_ts - created_ts).total_seconds())
802
+ if delta_seconds <= self.OPEN_EVENT_EPSILON_SECONDS:
803
+ return None
804
+ return event_ts
805
 
806
+ return None
 
807
 
808
+ def process_event(
809
+ self,
810
+ *,
811
+ event: dict,
812
+ event_ts,
813
+ year_start,
814
+ year_end,
815
+ state: dict,
816
+ ):
817
+ _ = event_ts
818
 
819
+ discussion = event.get("discussionData") or {}
820
+ if not isinstance(discussion, dict):
821
+ return
822
 
823
+ bucket_key, unique_id = self.classify_event(event, discussion)
824
+ if not bucket_key or not unique_id:
825
+ return
826
 
827
+ business_ts = self.extract_business_timestamp(bucket_key, event, discussion)
828
+ if business_ts is None:
829
+ return
830
 
831
+ if not (year_start <= business_ts <= year_end):
832
+ return
833
 
834
+ if unique_id in state["seen_ids"][bucket_key]:
835
+ return
836
 
837
+ state["seen_ids"][bucket_key].add(unique_id)
838
+ state["dates"][bucket_key].append(business_ts.tz_localize(None).date())
 
839
 
840
+ def build_bucket(self, key: str, dates: list, selected_year: int):
841
+ return {
842
+ "title": self.TITLE_MAP[key],
843
+ "fetched": True,
844
+ "dates": dates,
845
+ "count": len(dates),
846
+ "logs": f"Found {len(dates)} {self.LOG_LABEL_MAP[key]} in {selected_year}.",
847
+ }
848
 
849
+ def build_result(
850
+ self,
851
+ *,
852
+ username: str,
853
+ selected_year: int,
854
+ state: dict,
855
+ logs: list[str],
856
+ pages_fetched: int,
857
+ oldest_event_ts_seen,
858
+ seen_target_year: bool,
859
+ crossed_before_year_start: bool,
860
+ ):
861
+ logs = list(logs)
862
+ logs.append(f"Fetched {pages_fetched} page(s) from recent-activity.")
863
+ if oldest_event_ts_seen is not None:
864
+ logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
865
+ logs.append(f"Seen target year: {seen_target_year}.")
866
+ logs.append(f"Crossed before year start: {crossed_before_year_start}.")
867
+ logs.append(
868
+ " | ".join(
869
+ f"{self.TITLE_MAP[key]}: {len(state['dates'][key])}"
870
+ for key in ["pr_opened", "pr_activity", "thread_opened", "discussion_activity"]
871
+ )
872
+ )
873
 
874
+ combined_logs = "\n\n".join(
875
+ [
876
+ "[Pull Requests]\n" + self.build_bucket("pr_opened", state["dates"]["pr_opened"], selected_year)["logs"],
877
+ "[PR Activity]\n" + self.build_bucket("pr_activity", state["dates"]["pr_activity"], selected_year)["logs"],
878
+ "[Community Threads]\n" + self.build_bucket("thread_opened", state["dates"]["thread_opened"], selected_year)["logs"],
879
+ "[Community Activity]\n" + self.build_bucket("discussion_activity", state["dates"]["discussion_activity"], selected_year)["logs"],
880
+ "[Fetch]\n" + "\n".join(logs),
881
+ ]
882
+ )
883
 
884
+ return {
885
+ "username": username,
886
+ "selected_year": selected_year,
887
+ "fetched": True,
888
+ "pr_opened": self.build_bucket("pr_opened", state["dates"]["pr_opened"], selected_year),
889
+ "pr_activity": self.build_bucket("pr_activity", state["dates"]["pr_activity"], selected_year),
890
+ "thread_opened": self.build_bucket("thread_opened", state["dates"]["thread_opened"], selected_year),
891
+ "discussion_activity": self.build_bucket("discussion_activity", state["dates"]["discussion_activity"], selected_year),
892
+ "logs": combined_logs,
893
+ }
894
 
895
+ def build_error_result(
896
+ self,
897
+ *,
898
+ username: str,
899
+ selected_year: int,
900
+ error,
901
+ logs: list[str],
902
+ ):
903
+ _ = logs
904
+ return {
905
  "username": username,
906
  "selected_year": selected_year,
907
  "fetched": True,
908
+ "pr_opened": build_failed_bucket("Pull Requests"),
909
+ "pr_activity": build_failed_bucket("PR Activity"),
910
+ "thread_opened": build_failed_bucket("Community Threads"),
911
+ "discussion_activity": build_failed_bucket("Community Activity"),
912
+ "logs": f"Community fetch failed: {str(error)}",
913
  }
914
 
 
 
 
 
 
 
 
915
 
916
+ class BucketedActivityProcessor:
917
+ def __init__(
918
+ self,
919
+ *,
920
+ bucket_titles: dict[str, str],
921
+ bucket_log_labels: dict[str, str],
922
+ classifier,
923
+ error_label: str,
924
+ ):
925
+ self.bucket_titles = bucket_titles
926
+ self.bucket_log_labels = bucket_log_labels
927
+ self.classifier = classifier
928
+ self.error_label = error_label
929
+
930
+ def init_state(self):
931
+ return {
932
+ "bucket_dates": {key: [] for key in self.bucket_titles},
933
+ "bucket_seen_ids": {key: set() for key in self.bucket_titles},
934
+ }
935
+
936
+ def process_event(
937
+ self,
938
+ *,
939
+ event: dict,
940
+ event_ts,
941
+ year_start,
942
+ year_end,
943
+ state: dict,
944
+ ):
945
+ if event_ts is None:
946
+ return
947
+
948
+ if not (year_start <= event_ts <= year_end):
949
+ return
950
+
951
+ bucket_key, unique_id = self.classifier(event)
952
+
953
+ if not bucket_key or not unique_id:
954
+ return
955
+
956
+ if bucket_key not in state["bucket_dates"]:
957
+ return
958
+
959
+ if unique_id in state["bucket_seen_ids"][bucket_key]:
960
+ return
961
+
962
+ state["bucket_seen_ids"][bucket_key].add(unique_id)
963
+ state["bucket_dates"][bucket_key].append(event_ts.tz_localize(None).date())
964
+
965
+ def build_result(
966
+ self,
967
+ *,
968
+ username: str,
969
+ selected_year: int,
970
+ state: dict,
971
+ logs: list[str],
972
+ pages_fetched: int,
973
+ oldest_event_ts_seen,
974
+ seen_target_year: bool,
975
+ crossed_before_year_start: bool,
976
+ ):
977
+ logs = list(logs)
978
  logs.append(f"Fetched {pages_fetched} page(s) from recent-activity.")
979
  if oldest_event_ts_seen is not None:
980
  logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
 
982
  logs.append(f"Crossed before year start: {crossed_before_year_start}.")
983
  logs.append(
984
  " | ".join(
985
+ f"{self.bucket_titles[key]}: {len(state['bucket_dates'][key])}"
986
+ for key in self.bucket_titles
987
  )
988
  )
989
 
990
+ result = {
991
+ "username": username,
992
+ "selected_year": selected_year,
993
+ "fetched": True,
994
+ "logs": "\n".join(logs),
995
+ }
996
+
997
+ for key, title in self.bucket_titles.items():
998
+ dates = state["bucket_dates"][key]
999
+ result[key] = build_fetched_bucket(
1000
+ title=title,
1001
+ dates=dates,
1002
+ selected_year=selected_year,
1003
+ label=self.bucket_log_labels[key],
1004
+ )
1005
 
 
1006
  return result
1007
 
1008
+ def build_error_result(
1009
+ self,
1010
+ *,
1011
+ username: str,
1012
+ selected_year: int,
1013
+ error,
1014
+ logs: list[str],
1015
+ ):
1016
+ _ = logs
1017
+ result = {
1018
  "username": username,
1019
  "selected_year": selected_year,
1020
  "fetched": True,
1021
+ "logs": f"{self.error_label} fetch failed: {str(error)}",
1022
  }
1023
 
1024
+ for key, title in self.bucket_titles.items():
1025
+ result[key] = build_failed_bucket(title)
1026
 
1027
+ return result
1028
 
1029
 
1030
+ # ============================================================
1031
+ # Activity classifiers + config
1032
+ # ============================================================
1033
  def classify_upvote_event(event: dict):
1034
  target_type = event.get("targetType")
1035
  target = event.get("target") or {}
 
1180
  )
1181
 
1182
 
1183
+ # ============================================================
1184
+ # Public activity fetch functions
1185
+ # ============================================================
1186
+ def fetch_all_community_activity(username: str, selected_year: int):
1187
+ username = (username or "").strip()
1188
+ selected_year = int(selected_year)
1189
+
1190
+ cache_key = (username.lower(), selected_year, "community")
1191
+ cached = get_cache(cache_key)
1192
+ if cached:
1193
+ return cached
1194
+
1195
+ if not username:
1196
+ state = empty_community_state(selected_year)
1197
+ state["fetched"] = True
1198
+ state["logs"] = "No username provided."
1199
+ set_cache(cache_key, state)
1200
+ return state
1201
+
1202
+ processor = CommunityActivityProcessor(username=username)
1203
+
1204
+ result = collect_recent_activity(
1205
  username=username,
1206
  selected_year=selected_year,
1207
+ activity_type="discussion",
1208
+ processor=processor,
1209
+ build_session_fn=build_session,
1210
+ rate_limiter_obj=recent_activity_rate_limiter,
1211
+ )
1212
+
1213
+ set_cache(cache_key, result)
1214
+ return result
1215
+
1216
+
1217
+ def fetch_single_section_activity(section_key: str, username: str, selected_year: int):
1218
+ username = (username or "").strip()
1219
+ selected_year = int(selected_year)
1220
+ cfg = SINGLE_SECTION_CONFIG[section_key]
1221
+
1222
+ cache_key = (username.lower(), selected_year, cfg["cache_scope"])
1223
+ cached = get_cache(cache_key)
1224
+ if cached:
1225
+ return cached
1226
+
1227
+ if not username:
1228
+ state = empty_single_section_state(section_key, selected_year)
1229
+ state["fetched"] = True
1230
+ state["logs"] = "No username provided."
1231
+ set_cache(cache_key, state)
1232
+ return state
1233
+
1234
+ processor = BucketedActivityProcessor(
1235
  bucket_titles=cfg["bucket_titles"],
1236
  bucket_log_labels=cfg["bucket_log_labels"],
 
1237
  classifier=cfg["classifier"],
1238
+ error_label=cfg["cache_scope"].capitalize(),
1239
  )
1240
 
1241
+ result = collect_recent_activity(
1242
+ username=username,
1243
+ selected_year=selected_year,
1244
+ activity_type=cfg["activity_type"],
1245
+ processor=processor,
1246
+ build_session_fn=build_session,
1247
+ rate_limiter_obj=recent_activity_rate_limiter,
1248
+ )
1249
+
1250
+ set_cache(cache_key, result)
1251
+ return result
1252
+
1253
 
1254
  # ============================================================
1255
  # Contribution grid helpers