Upload folder using huggingface_hub

app.py

@@ -713,7 +713,36 @@ async def handle_background_polling(mentor_history, planning_history):
 
     return updated_mentor, updated_planning
 
-async def handle_chat(message, history, persona="planning"):
+async def fetch_readme(repo_url_input, branch="main"):
+    """Fetch README.md from a GitHub repository."""
+    if not repo_url_input:
+        return ""
+    
+    owner, repo = parse_github_repo_id(repo_url_input)
+    if not repo:
+        return ""
+    if not owner:
+        owner = "JsonLord"
+        
+    github_token = get_github_token(owner)
+    # Use raw content to avoid base64 decoding manually if possible, or just use the standard API
+    headers = {"Authorization": f"token {github_token}", "Accept": "application/vnd.github.v3.raw"}
+    
+    # Try different common README names
+    for name in ["README.md", "README", "readme.md"]:
+        url = f"https://api.github.com/repos/{owner}/{repo}/contents/{name}?ref={branch}"
+        try:
+            response = requests.get(url, headers=headers)
+            if response.status_code == 200:
+                logger.info(f"Successfully fetched {name} for {owner}/{repo} (branch: {branch})")
+                return response.text
+        except Exception as e:
+            logger.error(f"Error fetching {name} for {owner}/{repo}: {e}")
+            
+    logger.warning(f"Could not find README for {owner}/{repo} (branch: {branch})")
+    return ""
+
+async def handle_chat(message, history, persona="planning", readme_content=""):
     """Handle chat using run_manual_agent directly to support streaming tool calls."""
     all_tools = await get_all_tools(include_jules=False)
     
@@ -739,10 +768,15 @@ async def handle_chat(message, history, persona="planning"):
     base_system_message = MENTOR_SYSTEM_PROMPT if persona == "mentor" else PLANNING_SYSTEM_PROMPT
     p_state = json.dumps(initial_project_state)
     
+    readme_context = ""
+    if readme_content:
+        readme_context = f"\n\n### ORIGINAL PROJECT CONTEXT (README.md)\nYou are helping to adapt and improve the following existing project:\n{readme_content}"
+
     system_message = (
         f"{base_system_message}\n\n"
         f"Long-Term Memory/Context: {ltm_context}\n"
         f"Project State: {p_state}"
+        f"{readme_context}"
     )
     
     lc_messages = convert_to_langchain_messages(history)
@@ -768,7 +802,7 @@ async def handle_chat(message, history, persona="planning"):
     # Update memory at the end
     await update_memory(long_term_store, namespace, lc_messages + [AIMessage(content=final_output)], ltm_context)
 
-async def handle_close_ideate(history, persona="planning"):
+async def handle_close_ideate(history, persona="planning", readme_content="", repo_url="", branch="main"):
     if not history:
         folders = get_ideation_logs()
         # Returns: chat_col, structured_col, desc, tasks, repos, expect, api, prof, space, token, status, dropdown, dropdown_comm
@@ -778,7 +812,7 @@ async def handle_close_ideate(history, persona="planning"):
 
     # 1. Extraction using alias-large (granular requests)
     fields_config = {
-        "project_description": "Extract the Project Description, including the vision, concrete goals, future use cases, and potential integrations.",
+        "project_description": "Based on the original README (if provided) and the chat log, extract the Project Description. Focus on how the project is being adapted and improved. Include the vision, concrete goals, future use cases, and potential integrations.",
         "tasks_tests": """Extract the Tasks and Tests following this EXACT 7-point template:
 1. Estimation of Project Scope from 1-10 and with a presentation of the core parts
 2. Project Description w/ vision for the project, concrete goals what it should be capable of, and future use cases, and future integrations into other projects, 
@@ -804,7 +838,11 @@ async def handle_close_ideate(history, persona="planning"):
     
     for key, instruction in fields_config.items():
         try:
-            prompt = f"{instruction}\n\nLOG:\n{chat_text}\n\nRespond ONLY with the extracted content for this field. Do not add conversational filler."
+            readme_context = ""
+            if readme_content:
+                readme_context = f"ORIGINAL PROJECT README:\n{readme_content}\n\n"
+            
+            prompt = f"{readme_context}{instruction}\n\nCHAT LOG:\n{chat_text}\n\nRespond ONLY with the extracted content for this field. Do not add conversational filler."
             logger.info(f"Extracting {key} for {persona}...")
             response = await ainvoke_with_retry(extraction_llm, prompt)
             content = response.content
@@ -829,6 +867,8 @@ async def handle_close_ideate(history, persona="planning"):
             f.write(str(content))
 
     save_file("chat_log.txt", chat_text)
+    if readme_content:
+        save_file("original_readme.md", readme_content)
     save_file("project_description.txt", extracted.get("project_description", ""))
     save_file("tasks_tests.txt", extracted.get("tasks_tests", ""))
     save_file("github_repos.txt", extracted.get("github_repos", ""))
@@ -843,6 +883,14 @@ async def handle_close_ideate(history, persona="planning"):
     }
     with open(os.path.join(folder_path, "hf_deployment_data.json"), "w") as f:
         json.dump(hf_data, f)
+    
+    # Repo Info
+    repo_info = {
+        "repo_url": repo_url,
+        "branch": branch
+    }
+    with open(os.path.join(folder_path, "repo_info.json"), "w") as f:
+        json.dump(repo_info, f)
 
     folders = get_ideation_logs()
     
@@ -943,9 +991,16 @@ async def handle_github_prep(idea_description, log_file, target_repo_input, hf_p
             if os.path.exists(src):
                 shutil.copy2(src, os.path.join(jules_temp_dir, fname))
 
-        # Write bundled context as well for compatibility
+        # Write Project_Context.md. Preference: original README, else bundled context.
+        final_context = context_md
+        readme_path = os.path.join(log_path, "original_readme.md")
+        if os.path.exists(readme_path):
+            with open(readme_path, "r") as f:
+                final_context = f.read()
+                logger.info("Using original_readme.md for Project_Context.md")
+
         with open(os.path.join(jules_temp_dir, "project_context.md"), 'w') as f:
-            f.write(context_md)
+            f.write(final_context)
 
         # 5. Generate/Append to AGENTS.md
         existing_agents_content = ""
@@ -1455,8 +1510,17 @@ async def update_fields_from_log(log_file):
                 if "space_name" in params and not space: space = params["space_name"]
                 
                 token = params.get("hf_token", "")
-                target_repo = params.get("target_repo", "")
-                github_owner = params.get("github_owner", "JsonLord")
+            except: pass
+
+        repo_info_path = os.path.join(log_path, "repo_info.json")
+        if os.path.exists(repo_info_path):
+            try:
+                with open(repo_info_path, "r") as f:
+                    rparams = json.load(f)
+                target_repo = rparams.get("repo_url", "")
+                # We can also update the default owner if we parse it
+                if "/" in target_repo:
+                    github_owner = target_repo.split("/")[-2]
             except: pass
     else:
         # Fallback to old file-based
@@ -1757,6 +1821,10 @@ with gr.Blocks() as demo:
 
     with gr.Tab("Mentor Session"):
         with gr.Column(visible=True) as mentor_chat_col:
+            with gr.Row():
+                mentor_repo_url = gr.Textbox(label="Project Repo (owner/repo or URL)", scale=3)
+                mentor_branch = gr.Dropdown(label="Branch", choices=["main"], value="main", scale=1, allow_custom_value=True)
+            mentor_readme_state = gr.State("")
             mentor_chatbot = gr.Chatbot()
             mentor_msg = gr.Textbox(label="Enter your idea or question")
             with gr.Row():
@@ -1780,15 +1848,19 @@ with gr.Blocks() as demo:
 
         mentor_status = gr.Textbox(label="Session Status", interactive=False)
 
-        async def mentor_respond(message, chat_history):
-            async for updated_history in handle_chat(message, chat_history, persona="mentor"):
+        async def mentor_respond(message, chat_history, readme):
+            async for updated_history in handle_chat(message, chat_history, persona="mentor", readme_content=readme):
                 yield "", updated_history
-        mentor_submit_btn.click(mentor_respond, [mentor_msg, mentor_chatbot], [mentor_msg, mentor_chatbot])
-        mentor_msg.submit(mentor_respond, [mentor_msg, mentor_chatbot], [mentor_msg, mentor_chatbot])
+        mentor_submit_btn.click(mentor_respond, [mentor_msg, mentor_chatbot, mentor_readme_state], [mentor_msg, mentor_chatbot])
+        mentor_msg.submit(mentor_respond, [mentor_msg, mentor_chatbot, mentor_readme_state], [mentor_msg, mentor_chatbot])
         mentor_back_btn.click(lambda: (gr.update(visible=True), gr.update(visible=False)), None, [mentor_chat_col, mentor_structured_col])
 
     with gr.Tab("Planning Session"):
         with gr.Column(visible=True) as planning_chat_col:
+            with gr.Row():
+                plan_repo_url = gr.Textbox(label="Project Repo (owner/repo or URL)", scale=3)
+                plan_branch = gr.Dropdown(label="Branch", choices=["main"], value="main", scale=1, allow_custom_value=True)
+            plan_readme_state = gr.State("")
             chatbot = gr.Chatbot()
             msg = gr.Textbox(label="Enter your idea or question")
             with gr.Row():
@@ -1812,12 +1884,12 @@ with gr.Blocks() as demo:
 
         ideate_status = gr.Textbox(label="Session Status", interactive=False)
 
-        async def respond(message, chat_history):
-            async for updated_history in handle_chat(message, chat_history, persona="planning"):
+        async def respond(message, chat_history, readme):
+            async for updated_history in handle_chat(message, chat_history, persona="planning", readme_content=readme):
                 yield "", updated_history
 
-        p_click = submit_btn.click(respond, [msg, chatbot], [msg, chatbot])
-        p_submit = msg.submit(respond, [msg, chatbot], [msg, chatbot])
+        p_click = submit_btn.click(respond, [msg, chatbot, plan_readme_state], [msg, chatbot])
+        p_submit = msg.submit(respond, [msg, chatbot, plan_readme_state], [msg, chatbot])
         plan_back_btn.click(lambda: (gr.update(visible=True), gr.update(visible=False)), None, [planning_chat_col, planning_structured_col])
 
     with gr.Tab("Push Files") as push_files_tab:
@@ -1921,15 +1993,22 @@ with gr.Blocks() as demo:
         nudge_btn.click(handle_supervisor_nudge, inputs=[mon_session_id, log_dropdown_comm], outputs=[nudge_output])
 
     # End Session Logic
-    async def mentor_close_wrap(h):
-        return await handle_close_ideate(h, persona="mentor")
+    async def mentor_close_wrap(h, readme):
+        return await handle_close_ideate(h, persona="mentor", readme_content=readme)
         
-    async def planning_close_wrap(h):
-        return await handle_close_ideate(h, persona="planning")
+    async def planning_close_wrap(h, readme):
+        return await handle_close_ideate(h, persona="planning", readme_content=readme)
+
+    # End Session Logic
+    async def mentor_close_wrap(h, readme, repo, branch):
+        return await handle_close_ideate(h, persona="mentor", readme_content=readme, repo_url=repo, branch=branch)
+        
+    async def planning_close_wrap(h, readme, repo, branch):
+        return await handle_close_ideate(h, persona="planning", readme_content=readme, repo_url=repo, branch=branch)
 
     mentor_close_btn.click(
         mentor_close_wrap, 
-        [mentor_chatbot], 
+        [mentor_chatbot, mentor_readme_state, mentor_repo_url, mentor_branch], 
         [mentor_chat_col, mentor_structured_col, mentor_proj_desc, mentor_tasks_tests, mentor_repos, mentor_expectations, mentor_endpoints, mentor_prof, mentor_space, mentor_token, mentor_status, log_dropdown, log_dropdown_comm]
     )
     
@@ -1937,13 +2016,26 @@ with gr.Blocks() as demo:
     
     close_btn.click(
         planning_close_wrap, 
-        [chatbot], 
+        [chatbot, plan_readme_state, plan_repo_url, plan_branch], 
         planning_outputs
     )
 
     # Planning session auto-pipeline (trigger close after final answer)
-    p_click.then(planning_close_wrap, [chatbot], planning_outputs)
-    p_submit.then(planning_close_wrap, [chatbot], planning_outputs)
+    p_click.then(planning_close_wrap, [chatbot, plan_readme_state, plan_repo_url, plan_branch], planning_outputs)
+    p_submit.then(planning_close_wrap, [chatbot, plan_readme_state, plan_repo_url, plan_branch], planning_outputs)
+
+    # Readme and Branch Loading Logic
+    async def update_readme_state(repo, branch):
+        content = await fetch_readme(repo, branch)
+        return content
+
+    mentor_repo_url.change(load_github_branches, inputs=[mentor_repo_url], outputs=[mentor_branch])
+    mentor_repo_url.change(update_readme_state, [mentor_repo_url, mentor_branch], [mentor_readme_state])
+    mentor_branch.change(update_readme_state, [mentor_repo_url, mentor_branch], [mentor_readme_state])
+    
+    plan_repo_url.change(load_github_branches, inputs=[plan_repo_url], outputs=[plan_branch])
+    plan_repo_url.change(update_readme_state, [plan_repo_url, plan_branch], [plan_readme_state])
+    plan_branch.change(update_readme_state, [plan_repo_url, plan_branch], [plan_readme_state])
 
     # Log Refresh Logic
     refresh_logs_btn_1.click(refresh_logs_ui, outputs=[log_dropdown, log_dropdown_comm])
@@ -1969,6 +2061,25 @@ with gr.Blocks() as demo:
         outputs=[repo_input]
     ).then(fn=None, js="() => { document.querySelectorAll('button').forEach(b => { if(b.textContent.includes('Jules Communication')) b.click(); }); }")
 
+    # Synchronization Logic for Repo and Branch across all tabs
+    def sync_repo_val(val):
+        return val, val, val, val
+    
+    def sync_branch_val(val):
+        return val, val, val, val
+
+    # Sync Repo URL
+    mentor_repo_url.change(sync_repo_val, [mentor_repo_url], [plan_repo_url, target_repo, repo_input, mentor_repo_url])
+    plan_repo_url.change(sync_repo_val, [plan_repo_url], [mentor_repo_url, target_repo, repo_input, plan_repo_url])
+    target_repo.change(sync_repo_val, [target_repo], [mentor_repo_url, plan_repo_url, repo_input, target_repo])
+    repo_input.change(sync_repo_val, [repo_input], [mentor_repo_url, plan_repo_url, target_repo, repo_input])
+
+    # Sync Branch
+    mentor_branch.change(sync_branch_val, [mentor_branch], [plan_branch, target_branch, branch_dropdown, mentor_branch])
+    plan_branch.change(sync_branch_val, [plan_branch], [mentor_branch, target_branch, branch_dropdown, plan_branch])
+    target_branch.change(sync_branch_val, [target_branch], [mentor_branch, plan_branch, branch_dropdown, target_branch])
+    branch_dropdown.change(sync_branch_val, [branch_dropdown], [mentor_branch, plan_branch, target_branch, branch_dropdown])
+
     with gr.Tab("Upload"):
         hf_space_id_in = gr.Textbox(label="Hugging Face Space ID ({hf_profileID}/{spaceID})")
         with gr.Row():

test_fetch_readme.py

@@ -0,0 +1,47 @@
+import asyncio
+import os
+import requests
+
+def parse_github_repo_id(input_str):
+    if not input_str: return "", ""
+    input_str = input_str.strip().strip("/")
+    if "github.com/" in input_str:
+        import re
+        match = re.search(r'github\.com/([^/\s]+)/([^/\s.]+)', input_str)
+        if match:
+            repo = match.group(2)
+            if repo.endswith(".git"): repo = repo[:-4]
+            return match.group(1), repo
+    if "/" in input_str:
+        parts = input_str.split("/")
+        if len(parts) >= 2:
+            repo = parts[-1]
+            if repo.endswith(".git"): repo = repo[:-4]
+            return parts[-2], repo
+    return "", input_str.replace(".git", "")
+
+async def fetch_readme(repo_url_input, branch="main"):
+    owner, repo = parse_github_repo_id(repo_url_input)
+    headers = {"Accept": "application/vnd.github.v3.raw"}
+    token = os.environ.get("GITHUB_TOKEN")
+    if token:
+        headers["Authorization"] = f"token {token}"
+    
+    url = f"https://api.github.com/repos/{owner}/{repo}/contents/README.md?ref={branch}"
+    print(f"Fetching {url}...")
+    response = requests.get(url, headers=headers)
+    if response.status_code == 200:
+        return response.text
+    else:
+        print(f"Status: {response.status_code}, Text: {response.text[:100]}")
+    return ""
+
+async def main():
+    content = await fetch_readme("langchain-ai/langchain")
+    if content:
+        print("Success!")
+    else:
+        print("Failed.")
+
+if __name__ == "__main__":
+    asyncio.run(main())