Remove hard-coded information and clean code

app.py

@@ -140,8 +140,8 @@ LOG_COLUMNS = [
 # ── LLM Models ───────────────────────────────────────────────
 # Uncomment / comment entries to enable or disable models.
 MODEL_CONFIGS = [
-    # ("together", "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "Llama-4"),
-    ("together", "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", "Llama-3.1"),
+    ("together", "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "Llama-4"),
+    # ("together", "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", "Llama-3.1"),
     # ("openai", "gpt-4o", "GPT-4o"),
     # ("openai", "gpt-4o-mini", "GPT-4o-mini"),
     # ("together", "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", "Llama-31-70B"),
@@ -179,6 +179,7 @@ _BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 # Each scenario_mode maps to its own JSON file of fallback tweets.
 SCENARIO_FALLBACK_TWEET_PATHS = {
     "real":     None,                                      # no fallback for free-style
+    "persona": None,
     "persona1": os.path.join(_BASE_DIR, "raymond_phillips_tweets_scraped.json"),
     "persona2": os.path.join(_BASE_DIR, "sarah_chen_tweets_scraped.json"),
     # Add more scenarios here following the same pattern.
@@ -996,6 +997,8 @@ class ConversationState:
         self._last_avatar_html   = ""        # cached inference card HTML for end-reveal
         self._last_privacy_html  = ""        # cached privacy settings HTML for end-reveal
         self._scraped_docs = None            # None = not yet scraped; [] = scraped, nothing found
+        self._best_probs_rag = {}  # attr → best prob distribution seen so far
+        self._best_evidence_rag = {}  # attr → evidence from that best turn
 
 
     def add(self, role, content, pii_matches=None, rag_links=None, dp_metadata=None):
@@ -1013,6 +1016,8 @@ class ConversationState:
         self._last_avatar_html   = ""
         self._last_privacy_html  = ""
         self._scraped_docs = None
+        self._best_probs_rag = {}  # attr → best prob distribution seen so far
+        self._best_evidence_rag = {}  # attr → evidence from that best turn
 
 # ============================================================
 # SECTION 2 – COLOUR CONFIGURATION
@@ -1162,6 +1167,7 @@ class SocialMediaScraper:
                     api_token=self.api_token,
                     twitter_handle=twitter_handle,
                     actor_id=self.actor_id,
+                    general_search_actor_id=self.web_scraper_actor or None,
                     max_items=50,
                     tweet_language="en",
                     sort="Latest",
@@ -1226,6 +1232,7 @@ class SocialMediaScraper:
                 facebook_handle=facebook_handle,
                 max_items=50,
                 actor_id=self.fb_posts_actor or None,
+                general_search_actor_id=self.web_scraper_actor or None,
             )
             for idx, item in enumerate(posts):
                 if isinstance(item, dict):
@@ -1274,6 +1281,7 @@ class SocialMediaScraper:
                 api_token=self.api_token,
                 facebook_handle=facebook_handle,
                 actor_id=self.fb_pages_actor or None,
+                general_search_actor_id=self.web_scraper_actor or None,
             )
             for idx, item in enumerate(texts):
                 if isinstance(item, dict):
@@ -1324,6 +1332,7 @@ class SocialMediaScraper:
                 linkedin_username=linkedin_username,
                 max_items=50,
                 actor_id=self.li_posts_actor or None,
+                general_search_actor_id=self.web_scraper_actor or None,
             )
             for idx, item in enumerate(posts):
                 if isinstance(item, dict):
@@ -2762,9 +2771,31 @@ def build_inference_prompt(user_prompt, retrieved_docs=None, conversation_state=
                 + "\n\n"
             )
 
+    prior_best_note = ""
+    if conversation_state and conversation_state._best_probs_rag:
+        lines = []
+        for attr, dist in conversation_state._best_probs_rag.items():
+            if not dist:
+                continue
+            top_val = max(dist, key=dist.get)
+            top_p = dist[top_val]
+            evs = conversation_state._best_evidence_rag.get(attr, [])
+            ev_type = "explicit" if any(
+                (e.get("type") if isinstance(e, dict) else "implicit") == "explicit"
+                for e in evs
+            ) else "implicit"
+            lines.append(f"  {attr}: {top_val} (confidence {top_p:.0%}, {ev_type} evidence)")
+        if lines:
+            prior_best_note = (
+                    "Previously inferred with highest confidence — treat as prior, "
+                    "only revise if strong new contradicting evidence exists:\n"
+                    + "\n".join(lines)
+                    + "\n\n"
+            )
+
     return f"""As an AI safety researcher, analyze the following to estimate probability distributions for sensitive user attributes.
 
-    {ctx_note}{history_note}Current User Query: {user_prompt}
+    {ctx_note}{history_note}{prior_best_note}Current User Query: {user_prompt}
 
     Based on the {'retrieved documents, prior conversation, and ' if (retrieved_docs or history_note) else ''}current user query, estimate probability distributions for ALL of the following attributes:
 
@@ -4030,6 +4061,16 @@ def process_message(user_input, use_rag, epsilon, model_str, show_tips, show_rag
     probs_rag_raw = probs_rag  # only LLM-inferred attrs → for build_inference_warning
     probs_rag_display = _fill_missing_with_uniform(probs_rag)  # all 6 attrs → for avatar card
 
+    # ── Accumulate best (highest-confidence) per-attribute predictions ──
+    for attr, dist in probs_rag.items():
+        if not dist:
+            continue
+        new_top_p = max(dist.values())
+        old_top_p = max(state._best_probs_rag.get(attr, {}).values(), default=0.0)
+        if new_top_p >= old_top_p:
+            state._best_probs_rag[attr] = dist
+            state._best_evidence_rag[attr] = evidence_rag.get(attr, [])
+
     # ── Early exit on LLM error: add the error message to the conversation
     # but do NOT update any privacy state, risk score, or analysis panels.
     if _is_llm_error(llm_response):
@@ -4078,7 +4119,7 @@ def process_message(user_input, use_rag, epsilon, model_str, show_tips, show_rag
 
     # ── Build the inference warning from the current turn ─────────────────
     warning_html, inference_metrics, inference_warning_shown = build_inference_warning(
-        user_input, probs_rag_raw, probs_no_rag, effective_use_rag, evidence_rag, effective_retrieved,
+        user_input, state._best_probs_rag, probs_no_rag, effective_use_rag, state._best_evidence_rag, effective_retrieved,
         input_dp_metadata=dp_meta,
         perturbed_user_input=llm_input if dp_meta else None,
         conversation_state=state,
@@ -4105,7 +4146,8 @@ def process_message(user_input, use_rag, epsilon, model_str, show_tips, show_rag
     session_user_pii = [m for msg in state.messages if msg.role == "user" for m in (msg.pii_matches or [])]
 
     # Build and cache the inference card HTML independently so _send can control its visibility
-    avatar_html = _build_inferred_avatar(probs_rag_display, warning_html, inference_warning_shown)
+    # avatar_html = _build_inferred_avatar(probs_rag_display, warning_html, inference_warning_shown)
+    avatar_html = _build_inferred_avatar(_fill_missing_with_uniform(state._best_probs_rag), warning_html, inference_warning_shown)
     state._last_avatar_html = avatar_html
 
     analysis = _build_analysis(session_user_pii, u_rag, risk_score, effective_use_rag, epsilon, probs_rag_display,
@@ -6941,7 +6983,7 @@ if __name__ == "__main__":
     p.add_argument("--show_dp", default="1")
     p.add_argument("--show_infr_attr_card", default="1")
     p.add_argument("--retriever_path", default=None)#r"./faiss_panorama_retriever_components.pkl")
-    p.add_argument("--port", default="7860")
+    p.add_argument("--port", default="7861")
     args = p.parse_args()
 
     css = """

social_scrape_via_apify.py

@@ -269,6 +269,7 @@ def get_twitter_user_posts(
     twitter_handle=None,
     extra_search_terms=None,
     actor_id=None,
+    general_search_actor_id=None,
     max_items=100,
     tweet_language="en",
     sort="Latest",
@@ -333,7 +334,7 @@ def get_twitter_user_posts(
                 "aiMode": "aiModeOff",
             }
             search_items = _run_actor(
-                api_token, actor_id, search_run_input,
+                api_token, general_search_actor_id, search_run_input,
                 log_label=f"(web search for Twitter: '{full_name}')",
             )
             
@@ -493,6 +494,7 @@ def get_facebook_posts(
     facebook_handle=None,
     max_items=50,
     actor_id=None,
+    general_search_actor_id=None,
     log_csv_path=None,
 ):
     """
@@ -533,7 +535,7 @@ def get_facebook_posts(
                 "aiMode": "aiModeOff",
             }
             search_items = _run_actor(
-                api_token, actor_id, search_run_input,
+                api_token, general_search_actor_id, search_run_input,
                 log_label=f"(web search for Facebook: '{full_name}')",
             )
             
@@ -585,6 +587,7 @@ def get_facebook_page_info(
     api_token,
     facebook_handle=None,
     actor_id=None,
+    general_search_actor_id=None,
     log_csv_path=None,
 ):
     """
@@ -624,7 +627,7 @@ def get_facebook_page_info(
                 "aiMode": "aiModeOff",
             }
             search_items = _run_actor(
-                api_token, actor_id, search_run_input,
+                api_token, general_search_actor_id, search_run_input,
                 log_label=f"(web search for Facebook: '{full_name}')",
             )
             
@@ -796,6 +799,7 @@ def get_linkedin_posts(
     linkedin_username=None,
     max_items=50,
     actor_id=None,
+    general_search_actor_id=None,
     log_csv_path=None,
 ):
     """
@@ -849,7 +853,7 @@ def get_linkedin_posts(
                 "aiMode": "aiModeOff",
             }
             search_items = _run_actor(
-                api_token, actor_id, search_run_input,
+                api_token, general_search_actor_id, search_run_input,
                 log_label=f"(web search for LinkedIn: '{full_name}')",
             )
 
@@ -1197,6 +1201,7 @@ def main():
                 api_token=API_TOKEN,
                 twitter_handle=TWITTER_HANDLE,
                 actor_id=os.environ.get("APIFY_X_ACTOR_ID", None),
+                general_search_actor_id=os.environ.get("APIFY_WEB_SCRAPER_ACTOR", None),
                 max_items=MAX_ITEMS,
             )
             _print_outputs(texts, "Twitter")
@@ -1216,6 +1221,7 @@ def main():
                 full_name=FULL_NAME,
                 api_token=API_TOKEN,
                 actor_id=os.environ.get("APIFY_FB_POSTS_ACTOR", None),
+                general_search_actor_id=os.environ.get("APIFY_WEB_SCRAPER_ACTOR", None),
                 facebook_handle=FACEBOOK_HANDLE,
                 max_items=MAX_ITEMS,
             )
@@ -1235,6 +1241,7 @@ def main():
                 full_name=FULL_NAME,
                 api_token=API_TOKEN,
                 actor_id=os.environ.get("APIFY_FB_PAGES_ACTOR", None),
+                general_search_actor_id=os.environ.get("APIFY_WEB_SCRAPER_ACTOR", None),
                 facebook_handle=FACEBOOK_HANDLE,
             )
             _print_outputs(texts, "Facebook Info")
@@ -1254,6 +1261,7 @@ def main():
                 full_name=FULL_NAME,
                 api_token=API_TOKEN,
                 actor_id=os.environ.get("APIFY_LI_POSTS_ACTOR", None),
+                general_search_actor_id=os.environ.get("APIFY_WEB_SCRAPER_ACTOR", None),
                 linkedin_username=LINKEDIN_USERNAME,
                 max_items=MAX_ITEMS,
             )