optional ai cleaning and add md link regex cleaning by default

app.ipynb

@@ -142,7 +142,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 41,
    "id": "4f486d3a",
    "metadata": {},
    "outputs": [],
@@ -168,7 +168,8 @@
     "# from cartesia.tts import CartesiaTTS\n",
     "import cartesia\n",
     "import requests\n",
-    "import urllib"
+    "import urllib\n",
+    "import re"
    ]
   },
   {
@@ -810,7 +811,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 42,
+   "id": "c5b0156a-f6d4-480a-b7b5-b0899e7520b9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def remove_urls_from_markdown(text):\n",
+    "    return re.sub(r'\\[([^\\]]+)\\]\\([^\\)]+\\)', r'\\1', text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
    "id": "db54a6a6-4bdc-430a-b1ea-444c249b77fb",
    "metadata": {},
    "outputs": [],
@@ -820,12 +833,12 @@
     "    # result = requests.get('https://r.jina.ai/'+urllib.parse.quote_plus(url))\n",
     "    result = requests.get('https://r.jina.ai/'+url)\n",
     "    result.raise_for_status()\n",
-    "    return result.text"
+    "    return remove_urls_from_markdown(result.text)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 47,
    "id": "75891855-6c08-4a42-9ad5-a02e0b43bb3d",
    "metadata": {},
    "outputs": [],
@@ -858,6 +871,8 @@
     "\n",
     "    idx = 0\n",
     "    while complete == False and idx < max_iters:\n",
+    "        print('Page Cleaning Iter:',idx)\n",
+    "        assert idx < max_iters\n",
     "        idx += 1\n",
     "        response = client.chat.completions.create(\n",
     "          model=\"gpt-4o\",\n",
@@ -895,19 +910,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 48,
    "id": "7899e7b2-beeb-40a4-a571-a2ccfc7c9618",
    "metadata": {},
    "outputs": [],
    "source": [
     "#| export\n",
-    "def get_page_text(url):\n",
-    "    return clean_page_md(get_page_md(url))"
+    "def get_page_text(url:str, ai_clean:bool):\n",
+    "    text = get_page_md(url)\n",
+    "    if ai_clean:\n",
+    "        text = clean_page_md(text)\n",
+    "    return text"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 50,
    "id": "e4fb3159-579b-4271-bc96-4cd1e2816eca",
    "metadata": {},
    "outputs": [],
@@ -918,9 +936,11 @@
     "    ### Define UI ###\n",
     "    gr.Markdown(\"# TTS\")\n",
     "    gr.Markdown(\"\"\"Start typing below and then click **Go** to create the speech from your text.\n",
-    "For requests longer than allowed by the API they will be broken into chunks automatically. [Spaces Link](https://matdmiller-tts-openai.hf.space/) | <a href=\"https://matdmiller-tts-openai.hf.space/\" target=\"_blank\">Spaces Link HTML</a>\"\"\")\n",
+    "For requests longer than allowed by the API they will be broken into chunks automatically. [Spaces Link](https://matdmiller-tts-openai.hf.space/) | <a href=\"https://matdmiller-tts-openai.hf.space/\" target=\"_blank\">Spaces Link HTML</a>\n",
+    "[https://r.jina.ai/](https://r.jina.ai/)\"\"\")\n",
     "    with gr.Row():\n",
     "        input_url = gr.Textbox(max_lines=1, label=\"Optional - Enter a URL\")\n",
+    "        input_clean_cb = gr.Checkbox(value=False, label='AI Clean Text')\n",
     "        get_url_content_btn = gr.Button(\"Get URL Contents\")\n",
     "    with gr.Row():\n",
     "        input_text = gr.Textbox(max_lines=100, label=\"Enter text here\")\n",
@@ -946,7 +966,7 @@
     "\n",
     "    ### Define UI Actions ###\n",
     "\n",
-    "    get_url_content_btn.click(fn=get_page_text, inputs=input_url, outputs=input_text)\n",
+    "    get_url_content_btn.click(fn=get_page_text, inputs=[input_url,input_clean_cb], outputs=input_text)\n",
     "    \n",
     "    # input_text \n",
     "    input_text.input(fn=get_input_text_len, inputs=input_text, outputs=input_text_length)\n",
@@ -982,7 +1002,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": 51,
    "id": "a00648a1-891b-470b-9959-f5d502055713",
    "metadata": {},
    "outputs": [],
@@ -996,7 +1016,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": 52,
    "id": "4b534fe7-4337-423e-846a-1bdb7cccc4ea",
    "metadata": {},
    "outputs": [
@@ -1025,16 +1045,9 @@
      "data": {
       "text/plain": []
      },
-     "execution_count": 37,
+     "execution_count": 52,
      "metadata": {},
      "output_type": "execute_result"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "TOKENS CLEANUP: 970\n"
-     ]
     }
    ],
    "source": [
@@ -1060,7 +1073,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 38,
+   "execution_count": 45,
    "id": "28e8d888-e790-46fa-bbac-4511b9ab796c",
    "metadata": {},
    "outputs": [
@@ -1090,7 +1103,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 40,
+   "execution_count": 53,
    "id": "0420310d-930b-4904-8bd4-3458ad8bdbd3",
    "metadata": {},
    "outputs": [],

app.py

@@ -6,7 +6,8 @@ __all__ = ['secret_import_failed', 'TEMP', 'TEMP_DIR', 'OPENAI_CLIENT_TTS_THREAD
            'launch_kwargs', 'queue_kwargs', 'verify_authorization', 'split_text', 'concatenate_audio',
            'create_speech_openai', 'create_speech_cartesiaai', 'create_speech', 'get_input_text_len',
            'get_generation_cost', 'get_model_choices', 'update_model_choices', 'get_voice_choices',
-           'update_voice_choices', 'split_text_as_md', 'get_page_md', 'clean_page_md', 'get_page_text']
+           'update_voice_choices', 'split_text_as_md', 'remove_urls_from_markdown', 'get_page_md', 'clean_page_md',
+           'get_page_text']
 
 # %% app.ipynb 4
 import os
@@ -72,6 +73,7 @@ import traceback
 import cartesia
 import requests
 import urllib
+import re
 
 # %% app.ipynb 11
 TEMP = os.environ.get('GRADIO_TEMP_DIR','/tmp/')
@@ -333,13 +335,17 @@ def split_text_as_md(*args, **kwargs):
     return '# Text Splits:\n' + '<br>----------<br>'.join(output)
 
 # %% app.ipynb 38
+def remove_urls_from_markdown(text):
+    return re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', text)
+
+# %% app.ipynb 39
 def get_page_md(url):
     # result = requests.get('https://r.jina.ai/'+urllib.parse.quote_plus(url))
     result = requests.get('https://r.jina.ai/'+url)
     result.raise_for_status()
-    return result.text
+    return remove_urls_from_markdown(result.text)
 
-# %% app.ipynb 40
+# %% app.ipynb 41
 # import json
 def clean_page_md(text):
     max_iters = 15
@@ -356,6 +362,8 @@ def clean_page_md(text):
 
     idx = 0
     while complete == False and idx < max_iters:
+        print('Page Cleaning Iter:',idx)
+        assert idx < max_iters
         idx += 1
         response = client.chat.completions.create(
           model="gpt-4o",
@@ -380,19 +388,24 @@ def clean_page_md(text):
 # res = clean_page_md(test_page_md)
 # res
 
-# %% app.ipynb 42
-def get_page_text(url):
-    return clean_page_md(get_page_md(url))
-
 # %% app.ipynb 43
+def get_page_text(url:str, ai_clean:bool):
+    text = get_page_md(url)
+    if ai_clean:
+        text = clean_page_md(text)
+    return text
+
+# %% app.ipynb 44
 with gr.Blocks(title='TTS', head='TTS', delete_cache=(3600,3600)) as app:
     
     ### Define UI ###
     gr.Markdown("# TTS")
     gr.Markdown("""Start typing below and then click **Go** to create the speech from your text.
-For requests longer than allowed by the API they will be broken into chunks automatically. [Spaces Link](https://matdmiller-tts-openai.hf.space/) | <a href="https://matdmiller-tts-openai.hf.space/" target="_blank">Spaces Link HTML</a>""")
+For requests longer than allowed by the API they will be broken into chunks automatically. [Spaces Link](https://matdmiller-tts-openai.hf.space/) | <a href="https://matdmiller-tts-openai.hf.space/" target="_blank">Spaces Link HTML</a>
+[https://r.jina.ai/](https://r.jina.ai/)""")
     with gr.Row():
         input_url = gr.Textbox(max_lines=1, label="Optional - Enter a URL")
+        input_clean_cb = gr.Checkbox(value=False, label='AI Clean Text')
         get_url_content_btn = gr.Button("Get URL Contents")
     with gr.Row():
         input_text = gr.Textbox(max_lines=100, label="Enter text here")
@@ -418,7 +431,7 @@ For requests longer than allowed by the API they will be broken into chunks auto
 
     ### Define UI Actions ###
 
-    get_url_content_btn.click(fn=get_page_text, inputs=input_url, outputs=input_text)
+    get_url_content_btn.click(fn=get_page_text, inputs=[input_url,input_clean_cb], outputs=input_text)
     
     # input_text 
     input_text.input(fn=get_input_text_len, inputs=input_text, outputs=input_text_length)
@@ -451,13 +464,13 @@ For requests longer than allowed by the API they will be broken into chunks auto
     
     
 
-# %% app.ipynb 44
+# %% app.ipynb 45
 # launch_kwargs = {'auth':('username',GRADIO_PASSWORD),
 #                  'auth_message':'Please log in to Mat\'s TTS App with username: username and password.'}
 launch_kwargs = {}
 queue_kwargs = {'default_concurrency_limit':10}
 
-# %% app.ipynb 46
+# %% app.ipynb 47
 #.py launch
 if __name__ == "__main__":
     app.queue(**queue_kwargs)