Update app.py

app.py

@@ -1,5 +1,7 @@
 import requests
 from gradio import Blocks, Button, Textbox, HTML
+from urllib.parse import urljoin, urlparse
+import re
 
 # Function to get meta tags from a URL
 def get_meta_tags(url):
@@ -21,111 +23,104 @@ def get_meta_tags(url):
             html = response.text
             data['meta_tags'] = parse_meta_tags(html)
 
-            sitemap_xml_link_tag = response.json().get("robot", {}).get("sitemap")
-            
-        except Exception as e:
-            print(f"An error occurred: {e}")
-    
-    except Exception as e:
-        print(f"An error occurred while fetching the URL: {e}")
+            # Find the favicon link (if exists)
+            favicon_url = extract_favicon(html, url)
+            data['favicon'] = favicon_url
 
-    return data
+            # Check if robots.txt exists
+            robots_txt_url = urljoin(url, "/robots.txt")
+            robots_response = requests.get(robots_txt_url)
+            if robots_response.status_code == 200:
+                data['robots_txt'] = True
+                data['robots_txt_content'] = robots_response.text
 
+            # Check if sitemap.xml exists
+            sitemap_url = urljoin(url, "/sitemap.xml")
+            sitemap_response = requests.get(sitemap_url)
+            if sitemap_response.status_code == 200:
+                data['sitemap_xml'] = True
+                data['sitemap_xml_content'] = sitemap_response.text
 
-# Function to parse meta tags from HTML
-def parse_meta_tags(html):
-    import re
+            return data
 
-    # Find all <meta> tags in the HTML and extract their attributes
-    patterns = [
-        r'<meta\s+(\w+)\s*=\s*"([^"]*)"',
-        r'<meta\s+(\S+)\s*\=(\S*)',
-        r'^<meta[^>]*>$'
-    ]
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        return {"error": str(e)}
 
+# Function to parse meta tags from HTML
+def parse_meta_tags(html):
     meta_tags = {}
-
-    for pattern in patterns:
-        matches = re.findall(pattern, html)
-        
-        # Adding the matched attribute-value pairs to a dictionary
-        if len(matches) > 1 and (len(matches[0]) == 2 or len(matches[1:]) == 3):
-            key_value_pairs = [(matches[i][0], matches[i+1] if i + 1 < len(matches[0]) else "") for i in range(0, len(matches), 2)]
-            
-        elif len(pattern) == 4:
-            # For the last pattern, there's no need to have any match of length >1
-            key_value_pairs = [(matches[i], matches[j]) for i,j in enumerate(zip(range(len(matches)-3),range(len(matches[0]))))]
-        
-        else: continue
-        
-        meta_tags.update({key: value for (key, value) in key_value_pairs if value})
-
+    # Use regex to find <meta> tags and extract attributes
+    matches = re.findall(r'<meta\s+([^\>]+)>', html)
+    for match in matches:
+        attrs = re.findall(r'(\w+)=["\']([^"\']+)["\']', match)
+        for attr in attrs:
+            meta_tags[attr[0]] = attr[1]
     return meta_tags
 
-
-# Function to join url with favicon link
-def urljoin(*parts):
-    from urllib.parse importurlunparse, urlparse
-
-    parsed = urlparse(url)
-    path_parts = []
-    
-    for part in parts:
-        if isinstance(part, tuple) and len(part) > 1:
-            scheme, rest = part
-            
-            # Handle protocol-less URLs with 'http' as default
-            if not scheme:
-                scheme = 'http'
-            
-            parsed = urlparse(url)
-        
-        path_parts.append(path_parts[-1] + "/" + str(part))
-    
-    return urlunparse((parsed.scheme, "", "/".join(path_parts), None, "", ""))
-
-# Function for the main interface.
+# Function to extract favicon URL from the HTML
+def extract_favicon(html, base_url):
+    # Look for the favicon in the HTML
+    match = re.search(r'<link\s+rel=["\']icon["\']\s+href=["\']([^"\']+)["\']', html)
+    if match:
+        favicon_url = match.group(1)
+        if not favicon_url.startswith('http'):
+            favicon_url = urljoin(base_url, favicon_url)
+        return favicon_url
+    return None
+
+# Function to format the result output
+def format_output(result):
+    if "error" in result:
+        return f"Error: {result['error']}"
+
+    output = "<h3>Meta Tags</h3>"
+    for key, value in result["meta_tags"].items():
+        output += f"<strong>{key}</strong>: {value}<br>"
+
+    if result['favicon']:
+        output += f"<h3>Favicon</h3><img src='{result['favicon']}' alt='Favicon' style='width:50px;height:50px;'><br>"
+    else:
+        output += "<h3>Favicon</h3><p>Missing</p><br>"
+
+    if result['robots_txt']:
+        output += "<h3>robots.txt</h3><p>Found</p><br>"
+        output += f"<pre>{result['robots_txt_content']}</pre><br>"
+    else:
+        output += "<h3>robots.txt</h3><p>Not found</p><br>"
+
+    if result['sitemap_xml']:
+        output += "<h3>sitemap.xml</h3><p>Found</p><br>"
+        output += f"<pre>{result['sitemap_xml_content']}</pre><br>"
+    else:
+        output += "<h3>sitemap.xml</h3><p>Not found</p><br>"
+
+    return output
+
+# Gradio Interface
 def get_meta_tags_ui():
-    html_output = Blocks()
-    
-    with html_output:
-        title_key = "Keys: "
-        
-        # Define a button
-        buttons = Button(
-            style="background-color: white",
-            text="Get Meta Tags from URL"
-        )
-        
-         # Add favicon to the interface
-        Buttons.append(Button(style='favicon', icon='link-icon'))
-
-         # Get meta tags
-        input_element = Textbox()
+    with Blocks() as interface:
+        # Input element to enter the URL
+        url_input = Textbox(label="Enter URL", placeholder="https://example.com")
         
+        # Button to trigger the meta tags fetch
+        lookup_button = Button("Get Meta Tags from URL")
 
-         def update(value):
-             if not value:
-                return
+        # HTML output area for the results
+        html_output = HTML()
 
-             result= get_meta_tags(value)
+        # Action when the button is clicked
+        def update(value):
+            if value:
+                result = get_meta_tags(value)
+                return format_output(result)
 
-             html_output.clear()
-
-             title_key = "Keys: "
-
-             output_html_string  = str(format_output(result))
-            
-            buttons.launch(input_element, submit=update)
-
-
-    interface = Blocks(
-        style="background-color: white",
-        elements=[html_output],
-    )
+        # Link button click with the update function
+        lookup_button.click(fn=update, inputs=url_input, outputs=html_output)
 
     return interface
 
-
+# Run the interface
 if __name__ == "__main__":
-    get_meta_tags_ui()
+    interface = get_meta_tags_ui()
+    interface.launch()