recency fix again

research_ai_agent.py

@@ -225,53 +225,31 @@ def setup_superlinked_minimal(df):
     max_date = df['published'].max()
     print(f"📅 Dataset date range: {min_date} to {max_date}")
     
-    # DYNAMIC RECENCY: Calculate periods based on actual data, not hardcoded dates
-    # This will work regardless of when the code is run
-    
-    # Calculate the age range in the dataset
-    current_time = pd.Timestamp.now()
-    df['age_days'] = (current_time - df['published']).dt.days
-    
-    min_age = df['age_days'].min()
-    max_age = df['age_days'].max()
-    
-    print(f"📊 Dataset age range: {min_age} to {max_age} days old")
-    print(f"📊 That's {min_age/365:.1f} to {max_age/365:.1f} years old")
-    
-    # Find the age difference between newest and oldest papers
-    age_span = max_age - min_age
-    
-    print(f"📊 Age span in dataset: {age_span} days ({age_span/365:.2f} years)")
-    
-    # Create periods that give graduated scoring based on relative age within the dataset
-    # Newer papers (closer to min_age) get more points
-    period_1 = min_age + (age_span * 0.25)  # Newest 25% of papers
-    period_2 = min_age + (age_span * 0.50)  # Newest 50% of papers  
-    period_3 = min_age + (age_span * 0.75)  # Newest 75% of papers
-    period_4 = max_age + 365                # All papers in dataset + buffer
-    
-    print(f"📊 Recency periods (days old):")
-    print(f"  Period 1: {period_1:.0f} days ({period_1/365:.2f} years)")
-    print(f"  Period 2: {period_2:.0f} days ({period_2/365:.2f} years)")
-    print(f"  Period 3: {period_3:.0f} days ({period_3/365:.2f} years)")
-    print(f"  Period 4: {period_4:.0f} days ({period_4/365:.2f} years)")
-    
     # Text similarity space
     text_space = sl.TextSimilaritySpace(
         text=sl.chunk(paper.text, chunk_size=1000, chunk_overlap=0),
         model="sentence-transformers/all-mpnet-base-v2"
     )
     
-    # ADAPTIVE RECENCY: Periods calculated from actual dataset age distribution
+    # CORRECT RECENCY: Following the official example pattern
+    # Expanded for historical dataset (1993-2025 = ~32 years)
     recency_space = sl.RecencySpace(
         timestamp=paper.published_unix,
         period_time_list=[
-            sl.PeriodTime(timedelta(days=period_1)),  # Newest 25% get extra point
-            sl.PeriodTime(timedelta(days=period_2)),  # Newest 50% get extra point
-            sl.PeriodTime(timedelta(days=period_3)),  # Newest 75% get extra point
-            sl.PeriodTime(timedelta(days=period_4)),  # All papers get base point
+            sl.PeriodTime(timedelta(days=365)),        # papers within 1 year
+            sl.PeriodTime(timedelta(days=2*365)),      # papers within 2 years
+            sl.PeriodTime(timedelta(days=3*365)),      # papers within 3 years
+            sl.PeriodTime(timedelta(days=5*365)),      # papers within 5 years
+            sl.PeriodTime(timedelta(days=10*365)),     # papers within 10 years
+            sl.PeriodTime(timedelta(days=15*365)),     # papers within 15 years
+            sl.PeriodTime(timedelta(days=20*365)),     # papers within 20 years
+            sl.PeriodTime(timedelta(days=25*365)),     # papers within 25 years
+            sl.PeriodTime(timedelta(days=30*365)),     # papers within 30 years
+            sl.PeriodTime(timedelta(days=31*365)),     # papers within 31 years
+            sl.PeriodTime(timedelta(days=31*365 + 120)), # papers within 31.33 years (includes Feb 1994)
+            sl.PeriodTime(timedelta(days=32*365)),     # papers within 32 years (includes both)
         ],
-        negative_filter=-2.0  # Penalty for papers older than dataset range
+        negative_filter=-0.25
     )
     
     # Create index
@@ -353,53 +331,32 @@ def setup_superlinked_ultrafast(df):
     max_date = df['published'].max()
     print(f"📅 Dataset date range: {min_date} to {max_date}")
     
-    # DYNAMIC RECENCY: Calculate periods based on actual data, not hardcoded dates
-    # This will work regardless of when the code is run
-    
-    # Calculate the age range in the dataset
-    current_time = pd.Timestamp.now()
-    df['age_days'] = (current_time - df['published']).dt.days
-    
-    min_age = df['age_days'].min()
-    max_age = df['age_days'].max()
-    
-    print(f"📊 Dataset age range: {min_age} to {max_age} days old")
-    print(f"📊 That's {min_age/365:.1f} to {max_age/365:.1f} years old")
-    
-    # Find the age difference between newest and oldest papers
-    age_span = max_age - min_age
-    
-    print(f"📊 Age span in dataset: {age_span} days ({age_span/365:.2f} years)")
-    
-    # Create periods that give graduated scoring based on relative age within the dataset
-    # Newer papers (closer to min_age) get more points
-    period_1 = min_age + (age_span * 0.25)  # Newest 25% of papers
-    period_2 = min_age + (age_span * 0.50)  # Newest 50% of papers  
-    period_3 = min_age + (age_span * 0.75)  # Newest 75% of papers
-    period_4 = max_age + 365                # All papers in dataset + buffer
-    
-    print(f"📊 Recency periods (days old):")
-    print(f"  Period 1: {period_1:.0f} days ({period_1/365:.2f} years)")
-    print(f"  Period 2: {period_2:.0f} days ({period_2/365:.2f} years)")
-    print(f"  Period 3: {period_3:.0f} days ({period_3/365:.2f} years)")
-    print(f"  Period 4: {period_4:.0f} days ({period_4/365:.2f} years)")
-    
     # Text similarity space
     text_space = sl.TextSimilaritySpace(
         text=sl.chunk(paper.text, chunk_size=1000, chunk_overlap=0),
         model="sentence-transformers/all-mpnet-base-v2"
     )
     
-    # ADAPTIVE RECENCY: Periods calculated from actual dataset age distribution
+    # CORRECT RECENCY: Following the official example pattern
+    # Expanded for historical dataset (1993-2025 = ~32 years)
+    # Added granular periods for 30-32 year range to differentiate 1993 vs 1994
     recency_space = sl.RecencySpace(
         timestamp=paper.published_unix,
         period_time_list=[
-            sl.PeriodTime(timedelta(days=period_1)),  # Newest 25% get extra point
-            sl.PeriodTime(timedelta(days=period_2)),  # Newest 50% get extra point
-            sl.PeriodTime(timedelta(days=period_3)),  # Newest 75% get extra point
-            sl.PeriodTime(timedelta(days=period_4)),  # All papers get base point
+            sl.PeriodTime(timedelta(days=365)),        # papers within 1 year
+            sl.PeriodTime(timedelta(days=2*365)),      # papers within 2 years
+            sl.PeriodTime(timedelta(days=3*365)),      # papers within 3 years
+            sl.PeriodTime(timedelta(days=5*365)),      # papers within 5 years
+            sl.PeriodTime(timedelta(days=10*365)),     # papers within 10 years
+            sl.PeriodTime(timedelta(days=15*365)),     # papers within 15 years
+            sl.PeriodTime(timedelta(days=20*365)),     # papers within 20 years
+            sl.PeriodTime(timedelta(days=25*365)),     # papers within 25 years
+            sl.PeriodTime(timedelta(days=30*365)),     # papers within 30 years
+            sl.PeriodTime(timedelta(days=31*365)),     # papers within 31 years
+            sl.PeriodTime(timedelta(days=31*365 + 120)), # papers within 31.33 years (includes Feb 1994)
+            sl.PeriodTime(timedelta(days=32*365)),     # papers within 32 years (includes both)
         ],
-        negative_filter=-2.0  # Penalty for papers older than dataset range
+        negative_filter=-0.25
     )
     
     # Create index with both spaces - following query_time_weights.ipynb pattern