remove possilbe query parameters from the link

app.py

@@ -18,6 +18,7 @@ from langchain.schema import (
     SystemMessage
 )
 import random
+from urllib.parse import urlparse, urlunparse
 
 set_api_key(st.secrets["ELEVENLABS_API_KEY"])
 crawling_api_key = st.secrets["CRAWLING_API_KEY"]
@@ -40,14 +41,13 @@ def get_llm(model_name, model_temperature, api_key, max_tokens=None):
                           openai_api_key=api_key)
 
 
-def is_valid_web_link(text):
-    # Regular expression pattern to match a valid URL
-    url_pattern = re.compile(
-        r"^(https?)://"
-        r"[\w\-]+(\.[\w\-]+)+"  # Domain name (e.g., www.example.com)
-        r"(:\d+)?(/[\w\-./?%&=]*)?$"  # Optional port and path
-    )
-    return bool(url_pattern.match(text))
+def is_valid_web_link(url):
+    parsed_url = urlparse(url)
+    cleaned_url = parsed_url._replace(query='')._replace(params='')
+    if parsed_url.scheme and parsed_url.netloc:
+        return urlunparse(cleaned_url)
+    else:
+        return None
 
 
 @st.cache_data
@@ -182,7 +182,7 @@ def get_query_params():
         if 'web_url' in params:
             web_url = params['web_url'][0]
             if len(web_url) > 0:
-                if is_valid_web_link(web_url):
+                if web_url := is_valid_web_link(web_url):
                     st.session_state.web_url = web_url
 
 
@@ -216,16 +216,16 @@ def main() -> None:
             st.caption(description)
             st.divider()
 
-            content_url = st.text_input(label='Paste your link, e.g. https://expresso.today',
+            web_url = st.text_input(label='Paste your link, e.g. https://expresso.today',
                                         label_visibility='collapsed',
                                         placeholder='Paste your link, e.g. https://expresso.today')
             col1, _, _, _, col2 = st.columns(5)
             col1.button("Doodle")
             if col2.button("Random Page"):
-                content_url = get_random_page()
-            if len(content_url) > 0:
-                if is_valid_web_link(content_url):
-                    st.session_state.web_url = content_url
+                web_url = get_random_page()
+            if len(web_url) > 0:
+                if web_url := is_valid_web_link(web_url):
+                    st.session_state.web_url = web_url
                     st.experimental_rerun()
                 else:
                     st.warning(