Update app.py

app.py

@@ -167,7 +167,8 @@ def generate_filename(prompt, response, file_type="md"):
     Generate filename with meaningful terms and short dense clips from prompt & response.
     The filename should be about 150 chars total, include high-info terms, and a clipped snippet.
     """
-    prefix = datetime.now().strftime("%y%m_%H%M") + "_"
+    timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+    prefix = f"{timestamp}_"
     combined = (prompt + " " + response).strip()
     info_terms = get_high_info_terms(combined, top_n=10)
     
@@ -193,6 +194,24 @@ def create_file(prompt, response, file_type="md"):
         f.write(prompt + "\n\n" + response)
     return filename
 
+def append_to_transcript(entry: dict):
+    """Append a new entry at the top of transcript.md"""
+    transcript_file = "transcript.md"
+    new_entry = f"1. **Input:** {entry['input']}\n\n   **Output:** {entry['output']}\n\n   **Files:**\n"
+    for file in entry['files']:
+        emoji = FILE_EMOJIS.get(file.split('.')[-1], '')
+        new_entry += f"   - {emoji} [{os.path.basename(file)}]({file})\n"
+    new_entry += "\n---\n\n"
+
+    if os.path.exists(transcript_file):
+        with open(transcript_file, 'r', encoding='utf-8') as f:
+            existing_content = f.read()
+    else:
+        existing_content = ""
+    
+    with open(transcript_file, 'w', encoding='utf-8') as f:
+        f.write(new_entry + existing_content)
+
 def get_download_link(file, file_type="zip"):
     """Generate download link for file"""
     with open(file, "rb") as f:
@@ -212,6 +231,7 @@ def get_download_link(file, file_type="zip"):
 def clean_for_speech(text: str) -> str:
     """Clean text for speech synthesis"""
     text = text.replace("\n", " ")
+    text = text.replace("\r", " ")
     text = text.replace("</s>", " ")
     text = text.replace("#", "")
     text = re.sub(r"\(https?:\/\/[^\)]+\)", "", text)
@@ -239,7 +259,7 @@ async def edge_tts_generate_audio(text, voice="en-US-AriaNeural", rate=0, pitch=
     rate_str = f"{rate:+d}%"
     pitch_str = f"{pitch:+d}Hz"
     communicate = edge_tts.Communicate(text, voice, rate=rate_str, pitch=pitch_str)
-    out_fn = generate_filename(text, text, file_type=file_format)
+    out_fn = generate_filename(text, text, file_format=file_format)
     await communicate.save(out_fn)
     return out_fn
 
@@ -317,9 +337,16 @@ def process_video_with_gpt(video_path, prompt):
 
 # 🤖 9. AI Model Integration
 
-def save_full_transcript(query, text):
+def save_full_transcript(query, text, files):
     """Save full transcript of Arxiv results as a file."""
-    create_file(query, text, "md")
+    # Sanitize query by replacing carriage returns and line feeds
+    sanitized_query = query.replace('\r', ' ').replace('\n', ' ').strip()
+    entry = {
+        'input': sanitized_query,
+        'output': text.replace('\r', ' ').replace('\n', ' ').strip(),
+        'files': files
+    }
+    append_to_transcript(entry)
 
 def parse_arxiv_refs(ref_text: str):
     """
@@ -388,6 +415,7 @@ def create_paper_audio_files(papers, input_question):
     """
     # Collect all content for combined summary
     combined_titles = []
+    created_files = []
 
     for paper in papers:
         try:
@@ -398,6 +426,7 @@ def create_paper_audio_files(papers, input_question):
             file_format = st.session_state['audio_format']
             full_file = speak_with_edge_tts(full_text, voice=st.session_state['tts_voice'], file_format=file_format)
             paper['full_audio'] = full_file
+            created_files.append(full_file)
 
             # Display the audio immediately after generation
             st.write(f"### {FILE_EMOJIS.get(file_format, '')} {os.path.basename(full_file)}")
@@ -417,6 +446,9 @@ def create_paper_audio_files(papers, input_question):
         st.write(f"### {FILE_EMOJIS.get(file_format, '')} Combined Summary Audio")
         play_and_download_audio(combined_file, file_type=file_format)
         papers.append({'title': 'Combined Summary', 'full_audio': combined_file})
+        created_files.append(combined_file)
+    
+    return created_files
 
 def display_papers(papers):
     """
@@ -460,8 +492,9 @@ def perform_ai_lookup(q, vocal_summary=True, extended_refs=False,
 
     # Parse and process papers
     papers = parse_arxiv_refs(refs)
+    created_files = []
     if papers:
-        create_paper_audio_files(papers, input_question=q)
+        created_files = create_paper_audio_files(papers, input_question=q)
         display_papers(papers)
     else:
         st.warning("No papers found in the response.")
@@ -470,7 +503,9 @@ def perform_ai_lookup(q, vocal_summary=True, extended_refs=False,
     st.write(f"**Total Elapsed:** {elapsed:.2f} s")
 
     # Save full transcript
-    create_file(q, result, "md")
+    create_md_output = create_file(q, result, "md")
+    created_files.append(create_md_output)
+    save_full_transcript(q, result, created_files)
     return result
 
 def process_with_gpt(text):
@@ -488,7 +523,9 @@ def process_with_gpt(text):
         )
         ans = c.choices[0].message.content
         st.write("GPT-4o: " + ans)
-        create_file(text, ans, "md")
+        create_md_output = create_file(text, ans, "md")
+        created_files = [create_md_output]
+        save_full_transcript(text, ans, created_files)
         st.session_state.messages.append({"role":"assistant","content":ans})
     return ans
 
@@ -506,16 +543,19 @@ def process_with_claude(text):
         )
         ans = r.content[0].text
         st.write("Claude-3.5: " + ans)
-        create_file(text, ans, "md")
+        create_md_output = create_file(text, ans, "md")
+        created_files = [create_md_output]
+        save_full_transcript(text, ans, created_files)
         st.session_state.chat_history.append({"user":text,"claude":ans})
     return ans
 
 # 📂 10. File Management
 def create_zip_of_files(md_files, mp3_files, wav_files, input_question):
     """Create zip with intelligent naming based on top 10 common words."""
-    # Exclude 'readme.md'
-    md_files = [f for f in md_files if os.path.basename(f).lower() != 'readme.md']
+    # Exclude 'transcript.md'
+    md_files = [f for f in md_files if os.path.basename(f).lower() != 'transcript.md']
     all_files = md_files + mp3_files + wav_files
+
     if not all_files:
         return None
 
@@ -537,7 +577,7 @@ def create_zip_of_files(md_files, mp3_files, wav_files, input_question):
     combined_content = " ".join(all_content)
     info_terms = get_high_info_terms(combined_content, top_n=10)
     
-    timestamp = datetime.now().strftime("%y%m_%H%M")
+    timestamp = datetime.now().strftime("%y%m%d%H%M%S")
     name_text = '_'.join(term.replace(' ', '-') for term in info_terms[:10])
     zip_name = f"{timestamp}_{name_text}.zip"
     
@@ -547,93 +587,134 @@ def create_zip_of_files(md_files, mp3_files, wav_files, input_question):
     
     return zip_name
 
-def load_files_for_sidebar():
-    """Load and group files for sidebar display"""
-    md_files = glob.glob("*.md")
-    mp3_files = glob.glob("*.mp3")
-    wav_files = glob.glob("*.wav")  # 🆕 Load WAV files
-
-    md_files = [f for f in md_files if os.path.basename(f).lower() != 'readme.md']
-    all_files = md_files + mp3_files + wav_files
-
-    groups = defaultdict(list)
-    for f in all_files:
-        # Treat underscores as spaces and split into words
-        words = os.path.basename(f).replace('_', ' ').split()
-        # Extract keywords from filename
-        keywords = get_high_info_terms(' '.join(words), top_n=5)
-        group_name = '_'.join(keywords) if keywords else 'Miscellaneous'
-        groups[group_name].append(f)
-
-    # Sort groups based on latest file modification time
-    sorted_groups = sorted(groups.items(), key=lambda x: max(os.path.getmtime(f) for f in x[1]), reverse=True)
-    return sorted_groups
-
-def extract_keywords_from_md(files):
-    """Extract keywords from markdown files"""
-    text = ""
-    for f in files:
-        if f.endswith(".md"):
-            c = open(f,'r',encoding='utf-8').read()
-            text += " " + c
-    return get_high_info_terms(text, top_n=5)
-
-def display_file_manager_sidebar(groups_sorted):
+def load_transcript():
+    """Load and parse transcript.md"""
+    transcript_file = "transcript.md"
+    if not os.path.exists(transcript_file):
+        return []
+    
+    with open(transcript_file, 'r', encoding='utf-8') as f:
+        content = f.read()
+    
+    entries = content.split('\n---\n\n')
+    parsed_entries = []
+    for entry in entries:
+        if not entry.strip():
+            continue
+        match_input = re.search(r'\*\*Input:\*\* (.+)', entry)
+        match_output = re.search(r'\*\*Output:\*\* (.+)', entry)
+        files = re.findall(r'- (.+) \[(.+)\]\((.+)\)', entry)
+        if match_input and match_output:
+            parsed_entries.append({
+                'input': match_input.group(1),
+                'output': match_output.group(1),
+                'files': [file[2] for file in files]
+            })
+    return parsed_entries
+
+def display_file_manager_sidebar():
     """Display file manager in sidebar"""
     st.sidebar.title("🎵 Audio & Docs Manager")
 
-    all_md = []
-    all_mp3 = []
-    all_wav = []  # 🆕 List to hold WAV files
-    for group_name, files in groups_sorted:
-        for f in files:
-            if f.endswith(".md"):
-                all_md.append(f)
-            elif f.endswith(".mp3"):
-                all_mp3.append(f)
-            elif f.endswith(".wav"):
-                all_wav.append(f)  # 🆕 Append WAV files
-
+    groups_sorted = load_transcript()
+    
     top_bar = st.sidebar.columns(4)  # 🆕 Adjusted columns to accommodate WAV
     with top_bar[0]:
         if st.button("🗑 DelAllMD"):
-            for f in all_md:
+            md_files = glob.glob("*.md")
+            md_files = [f for f in md_files if os.path.basename(f).lower() != 'transcript.md']
+            for f in md_files:
                 os.remove(f)
             st.session_state.should_rerun = True
     with top_bar[1]:
         if st.button("🗑 DelAllMP3"):
-            for f in all_mp3:
+            mp3_files = glob.glob("*.mp3")
+            for f in mp3_files:
                 os.remove(f)
             st.session_state.should_rerun = True
     with top_bar[2]:
         if st.button("🗑 DelAllWAV"):
-            for f in all_wav:
+            wav_files = glob.glob("*.wav")  # 🆕 Load WAV files
+            for f in wav_files:
                 os.remove(f)
             st.session_state.should_rerun = True
     with top_bar[3]:
         if st.button("⬇️ ZipAll"):
-            zip_name = create_zip_of_files(all_md, all_mp3, all_wav, input_question=st.session_state.get('last_query', ''))
+            md_files = glob.glob("*.md")
+            md_files = [f for f in md_files if os.path.basename(f).lower() != 'transcript.md']
+            mp3_files = glob.glob("*.mp3")
+            wav_files = glob.glob("*.wav")  # 🆕 Load WAV files
+            zip_name = create_zip_of_files(md_files, mp3_files, wav_files, input_question=st.session_state.get('last_query', ''))
             if zip_name:
                 st.sidebar.markdown(get_download_link(zip_name, file_type="zip"), unsafe_allow_html=True)
 
-    for group_name, files in groups_sorted:
-        keywords_str = group_name.replace('_', ' ') if group_name else "No Keywords"
-        with st.sidebar.expander(f"{FILE_EMOJIS.get('md', '')} {group_name} Files ({len(files)}) - KW: {keywords_str}", expanded=True):
+    for idx, entry in enumerate(groups_sorted, 1):
+        group_name = f"Query {idx}: {entry['input'][:50]}..."  # Truncate for display
+        with st.sidebar.expander(f"{FILE_EMOJIS.get('md', '')} {group_name}", expanded=False):
             c1,c2 = st.columns(2)
             with c1:
-                if st.button("👀ViewGrp", key="view_group_"+group_name):
-                    st.session_state.viewing_prefix = group_name
+                if st.button("👀ViewGrp", key="view_group_"+str(idx)):
+                    st.session_state.viewing_prefix = idx
             with c2:
-                if st.button("🗑DelGrp", key="del_group_"+group_name):
-                    for f in files:
-                        os.remove(f)
+                if st.button("🗑DelGrp", key="del_group_"+str(idx)):
+                    # Delete associated files
+                    for f in entry['files']:
+                        if os.path.exists(f):
+                            os.remove(f)
+                    # Reload transcript without this entry
+                    all_entries = load_transcript()
+                    del all_entries[idx-1]
+                    # Rewrite transcript.md
+                    transcript_file = "transcript.md"
+                    with open(transcript_file, 'w', encoding='utf-8') as tf:
+                        for i, e in enumerate(all_entries, 1):
+                            new_entry = f"1. **Input:** {e['input']}\n\n   **Output:** {e['output']}\n\n   **Files:**\n"
+                            for file in e['files']:
+                                emoji = FILE_EMOJIS.get(file.split('.')[-1], '')
+                                new_entry += f"   - {emoji} [{os.path.basename(file)}]({file})\n"
+                            new_entry += "\n---\n\n"
+                            tf.write(new_entry)
                     st.success(f"Deleted group {group_name}!")
                     st.session_state.should_rerun = True
 
-            for f in files:
+            for f in entry['files']:
                 fname = os.path.basename(f)
                 ctime = datetime.fromtimestamp(os.path.getmtime(f)).strftime("%Y-%m-%d %H:%M:%S")
-                st.write(f"**{fname}** - {ctime}")
+                st.markdown(f"**{fname}** - {ctime}")
+
+def extract_keywords_from_md(files):
+    """Extract keywords from markdown files"""
+    text = ""
+    for f in files:
+        if f.endswith(".md"):
+            c = open(f,'r',encoding='utf-8').read()
+            text += " " + c
+    return get_high_info_terms(text, top_n=5)
+
+def display_viewing_group(group_idx):
+    """Display the contents of a viewing group"""
+    groups_sorted = load_transcript()
+    if group_idx < 1 or group_idx > len(groups_sorted):
+        st.error("Invalid group selected.")
+        return
+    entry = groups_sorted[group_idx-1]
+    st.write("---")
+    st.write(f"**Viewing Group {group_idx}:** {entry['input']}")
+    for f in entry['files']:
+        fname = os.path.basename(f)
+        ext = os.path.splitext(fname)[1].lower().strip('.')
+        st.write(f"### {fname}")
+        if ext == "md":
+            content = open(f,'r',encoding='utf-8').read()
+            st.markdown(content)
+        elif ext == "mp3":
+            st.audio(f)
+        elif ext == "wav":
+            st.audio(f)  # 🆕 Handle WAV files
+        else:
+            st.markdown(get_download_link(f), unsafe_allow_html=True)
+    if st.button("❌ Close"):
+        st.session_state.viewing_prefix = None
 
 # 🎯 11. Main Application
 def main():
@@ -726,7 +807,8 @@ def main():
             result = perform_ai_lookup(q, vocal_summary=vocal_summary, extended_refs=extended_refs, 
                                        titles_summary=titles_summary, full_audio=full_audio)
             if full_transcript:
-                save_full_transcript(q, result)
+                # The transcript is already being handled in perform_ai_lookup
+                pass
 
         st.markdown("### Change Prompt & Re-Run")
         q_new = st.text_input("🔄 Modify Query:")
@@ -735,7 +817,8 @@ def main():
             result = perform_ai_lookup(q_new, vocal_summary=vocal_summary, extended_refs=extended_refs, 
                                        titles_summary=titles_summary, full_audio=full_audio)
             if full_transcript:
-                save_full_transcript(q_new, result)
+                # The transcript is already being handled in perform_ai_lookup
+                pass
 
     elif tab_main == "🎤 Voice":
         st.subheader("🎤 Voice Input")
@@ -795,34 +878,16 @@ def main():
             st.write("Select a file from the sidebar to edit.")
 
     # Load and display files in the sidebar
-    groups_sorted = load_files_for_sidebar()
-    display_file_manager_sidebar(groups_sorted)
-
-    if st.session_state.viewing_prefix and any(st.session_state.viewing_prefix == group for group, _ in groups_sorted):
-        st.write("---")
-        st.write(f"**Viewing Group:** {st.session_state.viewing_prefix}")
-        for group_name, files in groups_sorted:
-            if group_name == st.session_state.viewing_prefix:
-                for f in files:
-                    fname = os.path.basename(f)
-                    ext = os.path.splitext(fname)[1].lower().strip('.')
-                    st.write(f"### {fname}")
-                    if ext == "md":
-                        content = open(f,'r',encoding='utf-8').read()
-                        st.markdown(content)
-                    elif ext == "mp3":
-                        st.audio(f)
-                    elif ext == "wav":
-                        st.audio(f)  # 🆕 Handle WAV files
-                    else:
-                        st.markdown(get_download_link(f), unsafe_allow_html=True)
-                break
-        if st.button("❌ Close"):
-            st.session_state.viewing_prefix = None
+    display_file_manager_sidebar()
 
+    # Display viewing group if selected
+    if st.session_state.viewing_prefix:
+        display_viewing_group(st.session_state.viewing_prefix)
+
+    # Additional Markdown in sidebar
     markdownPapers = """
     
-    # Levels of AGI
+# Levels of AGI
 
 ## 1. Performance (rows) x Generality (columns)
 - **Narrow**  
@@ -897,7 +962,7 @@ def main():
     - *Reference:* Stockfish (2023). **Stockfish Chess Engine**. [Website](https://stockfishchess.org)  
 - **Artificial Superintelligence (ASI)**  
   - Not yet achieved  
-
+    
 
 # 🧬 Innovative Architecture of AlphaFold2: A Hybrid System
 
@@ -944,7 +1009,7 @@ def main():
 
     """
     st.sidebar.markdown(markdownPapers)
-    
+
     if st.session_state.should_rerun:
         st.session_state.should_rerun = False
         st.rerun()