Update app.py

app.py

@@ -1,360 +1,106 @@
 from flask import Flask, render_template, request, jsonify, send_file
-from huggingface_hub import HfApi
-import requests
-import base64
+from .core import create_markdown_document
 import markdown
-import json
-import mimetypes
 import os
-import io
-from pathlib import Path
+import pkg_resources
+import sys
 
-app = Flask(__name__)
-
-GITHUB_API = "https://api.github.com/repos/"
-
-def generate_file_tree(paths):
-    """Generate a simple file tree from a list of paths."""
-    print("generating file tree") 
-    tree = ["📁 Root"]
-    sorted_paths = sorted(paths)
-    for path in sorted_paths:
-        parts = path.split('/')
-        indent = "  " * (len(parts) - 1)
-        tree.append(f"{indent}📄 {parts[-1]}")
-    print("generating file tree - Complete") 
-    
-    return "\n".join(tree) + "\n\n"
-
-def get_all_files(owner, repo, path="", is_hf=False):
-    """Recursively fetch all files from a repository."""
-    if is_hf:
-        api_url = f"https://huggingface.co/api/spaces/{owner}/{repo}/tree/main/{path}".rstrip('/')
-    else:
-        api_url = f"{GITHUB_API}{owner}/{repo}/contents/{path}".rstrip('/')
-    
+def find_template_path():
+    """Find the templates directory, either from installed package or source."""
     try:
-        response = requests.get(api_url, headers={"Accept": "application/json"}, timeout=10)
-        response.raise_for_status()
-        
-        # Check if the response is JSON
-        if response.headers.get('Content-Type', '').startswith('application/json'):
-            items = response.json()
+        # Try to use pkg_resources for installed packages
+        template_path = pkg_resources.resource_filename("repo_to_md", "templates")
+    except Exception as e:
+        # Fallback for running from source (development)
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        template_path = os.path.join(current_dir, "../templates")
+        template_path = os.path.abspath(template_path)
+
+    # Check if the template path exists
+    if not os.path.exists(template_path):
+        # Try an alternative path within the package directory
+        package_dir = os.path.dirname(os.path.abspath(__file__))
+        alternative_path = os.path.join(package_dir, "templates")
+        if os.path.exists(alternative_path):
+            template_path = alternative_path
         else:
-            print(f"Received non-JSON response from {api_url}: {response.text[:100]}...")
-            return None
-        
-        files = []
-        for item in items:
-            if isinstance(item, dict) and item.get('type') == 'file':
-                files.append(item)
-            elif isinstance(item, dict) and item.get('type') == 'dir':
-                sub_files = get_all_files(owner, repo, item['path'], is_hf)
-                if sub_files:
-                    files.extend(sub_files)
-        return files
-        
-    except requests.exceptions.RequestException as e:
-        print(f"Error fetching repository contents from {api_url}: {str(e)}")
-        return None
-
-def get_hf_files(repo, name):
-    """Fetch all files from a Hugging Face Space with robust error handling."""
-    api = HfApi(token=os.getenv('HF_TOKEN'))
-    try:
-        # Use HfApi to list files, which is more reliable for Spaces
-        file_list = api.list_repo_files(repo_id=f'{repo}/{name}', repo_type="space")
-        print(f"Files in {repo}/{name}: {file_list}")
-        processed_files = []
-        
-        if not os.path.exists(name):
-            os.makedirs(name)
-
-        for file_path in file_list:
-            # Fetch raw file content with strict validation
-            raw_url = f"https://huggingface.co/spaces/{repo}/{name}/raw/main/{file_path}"
-            try:
-                response = requests.get(raw_url, timeout=10)
-                response.raise_for_status()
-                
-                # Ensure we get raw content, not HTML or JSON
-                content_type = response.headers.get('Content-Type', '').lower()
-                if content_type.startswith('text/html'):
-                    print(f"Warning: Received HTML instead of raw content for {file_path}: {response.text[:100]}...")
-                    continue
-                if content_type.startswith('application/json'):
-                    print(f"Warning: Received JSON instead of raw content for {file_path}: {response.text[:100]}...")
-                    continue
-                
-                # Verify it's a valid file (e.g., text/plain or binary)
-                if not content_type.startswith(('text/plain', 'application/octet-stream', 'text/')) and 'text/' not in content_type:
-                    print(f"Unexpected content type for {file_path}: {content_type}")
-                    continue
-                
-            except requests.exceptions.RequestException as e:
-                print(f"Error downloading {file_path} from {raw_url}: {str(e)}")
-                continue
-            
-            # Process file
-            filename = os.path.basename(file_path)
-            if "." in filename:
-                pf, sf = filename.rsplit(".", 1)
-                f_name = f"{pf}.{sf}"
+            # One last attempt: check if installed but in a different structure
+            site_packages = os.path.join(os.path.dirname(sys.executable), "site-packages")
+            installed_path = os.path.join(site_packages, "repo_to_md", "templates")
+            if os.path.exists(installed_path):
+                template_path = installed_path
             else:
-                pf = filename
-                sf = ""
-                f_name = pf
-            
-            local_path = os.path.join(name, file_path)
-            os.makedirs(os.path.dirname(local_path), exist_ok=True)
-            
-            with open(local_path, 'wb') as file:
-                file.write(response.content)
-            
-            processed_files.append({"path": file_path})
+                raise FileNotFoundError(
+                    f"Template directory not found at: {template_path}, {alternative_path}, or {installed_path}"
+                )
 
-        print(f"Processed files: {processed_files}")
-        return processed_files
-    
-    except Exception as e:
-        print(f"Error processing Hugging Face files for {repo}/{name}: {str(e)}")
-        return []
+    return template_path
 
-def get_repo_contents(url):
-    """Parse URL and fetch repository contents with robust error handling."""
-    try:
-        if "huggingface.co" in url.lower():
-            parts = url.rstrip('/').split('/')
-            owner, repo = parts[-2], parts[-1]
-            # Ensure the Space exists and is accessible
-            try:
-                api = HfApi()
-                api.list_repo_files(repo_id=f'{owner}/{repo}', repo_type="space")  # Pre-check
-            except Exception as e:
-                raise Exception("HfApi Error")
+def run_demo(host="0.0.0.0", port=7860, debug=True):
+    # Get the correct template path
+    template_path = find_template_path()
 
-            files = get_hf_files(owner, repo)
-            if not files:  # Empty list is valid, but check for errors
-                raise Exception("No files found in the Hugging Face Space")
-            return owner, repo, files, True
-        else:  # Assume GitHub URL
-            parts = url.rstrip('/').split('/')
-            owner, repo = parts[-2], parts[-1]
-            files = get_all_files(owner, repo, "", False)
-            if files is None:
-                raise Exception("Failed to fetch GitHub repository contents")
-            return owner, repo, files, False
-    except Exception as e:
-        print(f"Error processing URL {url}: {str(e)}")
-        return None, None, f"Error fetching repo contents: {str(e)}", False
+    # Create Flask app with the resolved template folder
+    app = Flask(__name__, template_folder=template_path)
 
-def process_file_content(file_info, owner, repo, is_hf=False):
-    """Process individual file content from a repository."""
-    content = ""
-    file_path = file_info['path']
-    
-    try:
-        if is_hf:
-            file_url = f"https://huggingface.co/spaces/{owner}/{repo}/raw/main/{file_path}"
-            response = requests.get(file_url, timeout=10)
-            response.raise_for_status()
-            
-            # Ensure we get raw content, not HTML or JSON
-            content_type = response.headers.get('Content-Type', '').lower()
-            if content_type.startswith('text/html'):
-                raise Exception(f"Received HTML instead of raw content for {file_path}: {response.text[:100]}...")
-            if content_type.startswith('application/json'):
-                raise Exception(f"Received JSON instead of raw content for {file_path}: {response.text[:100]}...")
-            
-            content_raw = response.content
-            size = len(content_raw)
-            file_extension = file_path.split('.')[-1] if '.' in file_path else ''
-            mime_type, _ = mimetypes.guess_type(file_path)
-            is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json'] or "Dockerfile" in file_path
-            
-            if is_text:
-                try:
-                    text_content = content_raw.decode('utf-8')
-                    if file_extension == 'json':
-                        try:
-                            json_data = json.loads(text_content)
-                            formatted_json = json.dumps(json_data, indent=2)
-                            content = f"### File: {file_path}\n```json\n{formatted_json}\n```\n\n"
-                        except json.JSONDecodeError:
-                            content = f"### File: {file_path}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n"
-                    else:
-                        content = f"### File: {file_path}\n```{file_extension or 'text'}\n{text_content}\n```\n\n"
-                except UnicodeDecodeError:
-                    content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
-            else:
-                content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
-        else:  # GitHub
-            file_url = f"{GITHUB_API}{owner}/{repo}/contents/{file_path}"
-            response = requests.get(file_url, headers={"Accept": "application/json"}, timeout=10)
-            response.raise_for_status()
-            
-            # Ensure we get JSON, not HTML
-            if response.headers.get('Content-Type', '').startswith('text/html'):
-                raise Exception(f"Received HTML instead of JSON for {file_path}: {response.text[:100]}...")
-            
-            data = response.json()
-            if 'content' in data:
-                content_raw = base64.b64decode(data['content'])
-                size = data['size']
-                file_extension = file_path.split('.')[-1] if '.' in file_path else ''
-                mime_type, _ = mimetypes.guess_type(file_path)
-                is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json']
+    @app.route('/')
+    def index():
+        return render_template('index.html')
+
+    @app.route('/process', methods=['POST'])
+    def process():
+        response_data = {'markdown': '', 'html': '', 'filename': '', 'error': None}
+        
+        try:
+            if 'files[]' in request.files:
+                files = request.files.getlist('files[]')
+                if not files:
+                    response_data['error'] = 'No files uploaded'
+                    return jsonify(response_data), 400
                 
-                if is_text:
-                    try:
-                        text_content = content_raw.decode('utf-8')
-                        if file_extension == 'json':
-                            try:
-                                json_data = json.loads(text_content)
-                                formatted_json = json.dumps(json_data, indent=2)
-                                content = f"### File: {file_path}\n```json\n{formatted_json}\n```\n\n"
-                            except json.JSONDecodeError:
-                                content = f"### File: {file_path}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n"
-                        else:
-                            content = f"### File: {file_path}\n```{file_extension or 'text'}\n{text_content}\n```\n\n"
-                    except UnicodeDecodeError:
-                        content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
-                else:
-                    content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
+                markdown_content = create_markdown_document(files=files)
+                response_data['markdown'] = markdown_content
+                response_data['html'] = markdown.markdown(markdown_content)
+                response_data['filename'] = "uploaded_files_summary.md"
             else:
-                content = f"### File: {file_path}\n[No content available]\n\n"
-    except Exception as e:
-        content = f"### File: {file_path}\n[Error fetching file content: {str(e)}]\n\n"
-    
-    return content
-
-def process_uploaded_file(file):
-    """Process uploaded file content."""
-    content = ""
-    filename = file.filename
-    file_extension = filename.split('.')[-1] if '.' in filename else ''
-    
-    try:
-        content_raw = file.read()  # Read file content into memory
-        size = len(content_raw)    # Compute size in bytes
+                repo_url = request.json.get('repo_url', '').strip()
+                if not repo_url:
+                    response_data['error'] = 'Please provide a repository URL or upload files'
+                    return jsonify(response_data), 400
+                
+                markdown_content = create_markdown_document(repo_url)
+                if markdown_content.startswith("Error:"):
+                    response_data['error'] = markdown_content
+                    return jsonify(response_data), 400
+                
+                response_data['markdown'] = markdown_content
+                response_data['html'] = markdown.markdown(markdown_content)
+                owner, repo = repo_url.rstrip('/').split('/')[-2:]
+                response_data['filename'] = f"{owner}_{repo}_summary.md"
         
-        mime_type, _ = mimetypes.guess_type(filename)
-        is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json'] or "Dockerfile" in file_path
+        except Exception as e:
+            response_data['error'] = f"Server error processing request: {str(e)}"
+            return jsonify(response_data), 500
         
-        if is_text:
-            try:
-                text_content = content_raw.decode('utf-8')
-                if file_extension == 'json':
-                    try:
-                        json_data = json.loads(text_content)
-                        formatted_json = json.dumps(json_data, indent=2)
-                        content = f"### File: {filename}\n```json\n{formatted_json}\n```\n\n"
-                    except json.JSONDecodeError:
-                        content = f"### File: {filename}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n"
-                else:
-                    content = f"### File: {filename}\n```{file_extension or 'text'}\n{text_content}\n```\n\n"
-            except UnicodeDecodeError:
-                content = f"### File: {filename}\n[Binary file - {size} bytes]\n\n"
-        else:
-            content = f"### File: {filename}\n[Binary file - {size} bytes]\n\n"
-    except Exception as e:
-        content = f"### File: {filename}\n[Error processing file: {str(e)}]\n\n"
-    
-    return content
+        return jsonify(response_data)
 
-def create_markdown_document(url=None, files=None):
-    """Create markdown document from repo contents or uploaded files."""
-    if url:
-        owner, repo, contents, is_hf = get_repo_contents(url)
-        
-        if isinstance(contents, str):  # Error case
-            return f"Error: {contents}"
+    @app.route('/download', methods=['POST'])
+    def download():
+        markdown_content = request.json.get('markdown', '')
+        filename = request.json.get('filename', 'document.md')
         
-        markdown_content = f"# {'Space' if is_hf else 'Repository'}: {owner}/{repo}\n\n"
-        markdown_content += "## File Structure\n```\n"
-        markdown_content += generate_file_tree([item['path'] for item in contents])
-        markdown_content += "```\n\n"
-        markdown_content += f"Below are the contents of all files in the {'space' if is_hf else 'repository'}:\n\n"
+        buffer = io.BytesIO()
+        buffer.write(markdown_content.encode('utf-8'))
+        buffer.seek(0)
         
-        for item in contents:
-            markdown_content += process_file_content(item, owner, repo, is_hf)
-    else:  # Handle uploaded files
-        markdown_content = "# Uploaded Files\n\n"
-        markdown_content += "## File Structure\n```\n"
-        markdown_content += generate_file_tree([file.filename for file in files])
-        markdown_content += "```\n\n"
-        markdown_content += "Below are the contents of all uploaded files:\n\n"
-        for file in files:
-            markdown_content += process_uploaded_file(file)
-    
-    return markdown_content
-
-@app.route('/')
-def index():
-    return render_template('index.html')
-
-@app.route('/process', methods=['POST'])
-def process():
-    # Ensure consistent response structure as JSON, even for errors
-    response_data = {'markdown': '', 'html': '', 'filename': '', 'error': None}
-    
-    try:
-        if 'files[]' in request.files:
-            files = request.files.getlist('files[]')
-            if not files:
-                response_data['error'] = 'No files uploaded'
-                return jsonify(response_data), 400
-            
-            markdown_content = create_markdown_document(files=files)
-            response_data['markdown'] = "```markdown\n" + markdown_content + "\n```"
-            response_data['html'] = markdown.markdown(markdown_content)
-            response_data['filename'] = "uploaded_files_summary.md"
-        else:
-            repo_url = request.json.get('repo_url', '').strip()
-            if not repo_url:
-                response_data['error'] = 'Please provide a repository URL or upload files'
-                return jsonify(response_data), 400
-            
-            markdown_content = create_markdown_document(repo_url)
-            owner, repo, contents, is_hf = get_repo_contents(repo_url)
-            if not owner:
-                response_data['error'] = markdown_content
-                return jsonify(response_data), 400
-            
-            response_data['markdown'] = markdown_content
-            response_data['html'] = markdown.markdown(markdown_content)
-            response_data['filename'] = f"{owner}_{repo}_summary.md"
-    
-    except Exception as e:
-        response_data['error'] = f"Server error processing request: {str(e)}"
-        return jsonify(response_data), 500
-    
-    return jsonify(response_data)
-
-@app.route('/download', methods=['POST'])
-def download():
-    markdown_content = request.json.get('markdown', '')
-    filename = request.json.get('filename', 'document.md')
-    
-    buffer = io.BytesIO()
-    buffer.write(markdown_content.encode('utf-8'))
-    buffer.seek(0)
-    
-    return send_file(
-        buffer,
-        as_attachment=True,
-        download_name=filename,
-        mimetype='text/markdown'
-    )
-
-with open("html_template.html", "r") as f:
-    html_template = f.read()
-f.close()
+        return send_file(
+            buffer,
+            as_attachment=True,
+            download_name=filename,
+            mimetype='text/markdown'
+        )
 
-if not os.path.exists('templates'):
-    os.makedirs('templates')
-with open('templates/index.html', 'w') as f:
-    f.write(html_template)
+    app.run(host=host, port=port, debug=debug)
 
 if __name__ == '__main__':
-    app.run(host="0.0.0.0", port=7860, debug=True)
+    run_demo()