commit

app.py

@@ -0,0 +1,372 @@
+import gradio as gr
+from datasets import load_dataset
+import json
+import pandas as pd
+
+# Load the dataset
+dataset = load_dataset("danielrosehill/multimodal-ai-taxonomy")
+
+# Extract taxonomy data
+taxonomy_data = {}
+for split_name in dataset.keys():
+    if split_name.startswith("taxonomy_"):
+        # Parse the split name to get modality and operation type
+        parts = split_name.replace("taxonomy_", "").split("_")
+        if len(parts) >= 3:
+            modality_parts = parts[:-1]
+            operation = parts[-1]
+            modality = "_".join(modality_parts)
+
+            if modality not in taxonomy_data:
+                taxonomy_data[modality] = {}
+
+            # Get the modalities from this split
+            data = dataset[split_name]
+            if len(data) > 0:
+                taxonomy_data[modality][operation] = json.loads(data[0]['json'])
+
+# Define modality display names and emojis
+MODALITY_INFO = {
+    "video_generation": {"name": "Video Generation", "emoji": "🎬", "color": "#FF6B6B"},
+    "audio_generation": {"name": "Audio Generation", "emoji": "🎵", "color": "#4ECDC4"},
+    "image_generation": {"name": "Image Generation", "emoji": "🖼️", "color": "#95E1D3"},
+    "text_generation": {"name": "Text Generation", "emoji": "📝", "color": "#F38181"},
+    "3d_generation": {"name": "3D Generation", "emoji": "🎨", "color": "#AA96DA"},
+}
+
+# CSS for styling
+custom_css = """
+.modality-card {
+    border: 2px solid #e0e0e0;
+    border-radius: 10px;
+    padding: 20px;
+    margin: 10px 0;
+    background: white;
+    box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+}
+.modality-header {
+    font-size: 1.5em;
+    font-weight: bold;
+    margin-bottom: 10px;
+    color: #333;
+}
+.modality-meta {
+    background: #f5f5f5;
+    padding: 10px;
+    border-radius: 5px;
+    margin: 10px 0;
+}
+.badge {
+    display: inline-block;
+    padding: 4px 12px;
+    border-radius: 12px;
+    margin: 2px;
+    font-size: 0.85em;
+    font-weight: 500;
+}
+.badge-mature { background: #4CAF50; color: white; }
+.badge-emerging { background: #FF9800; color: white; }
+.badge-experimental { background: #9C27B0; color: white; }
+.index-card {
+    border: 2px solid #ddd;
+    border-radius: 15px;
+    padding: 30px;
+    margin: 15px;
+    text-align: center;
+    cursor: pointer;
+    transition: all 0.3s;
+    background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+    color: white;
+}
+.index-card:hover {
+    transform: translateY(-5px);
+    box-shadow: 0 10px 20px rgba(0,0,0,0.2);
+}
+.stat-box {
+    background: #f8f9fa;
+    border-radius: 10px;
+    padding: 15px;
+    margin: 10px;
+    text-align: center;
+}
+"""
+
+def create_modality_card(modality_obj):
+    """Create an HTML card for a single modality"""
+
+    # Maturity badge
+    maturity = modality_obj['metadata']['maturityLevel']
+    badge_class = f"badge badge-{maturity}"
+
+    # Input/Output info
+    input_primary = modality_obj['input']['primary']
+    input_secondary = modality_obj['input'].get('secondary', [])
+    output_primary = modality_obj['output']['primary']
+
+    # Build input string
+    input_str = f"**Primary:** {input_primary}"
+    if input_secondary:
+        input_str += f"<br>**Secondary:** {', '.join(input_secondary)}"
+
+    # Audio info for output
+    audio_info = ""
+    if modality_obj['output'].get('audio'):
+        audio_type = modality_obj['output'].get('audioType', 'N/A')
+        audio_info = f"<br>**Audio:** {audio_type}"
+
+    # Characteristics
+    chars = modality_obj.get('characteristics', {})
+    char_items = [f"**{k}:** {v}" for k, v in chars.items()]
+    char_str = "<br>".join(char_items) if char_items else "N/A"
+
+    # Use cases
+    use_cases = modality_obj['metadata'].get('commonUseCases', [])
+    use_case_str = "<br>• " + "<br>• ".join(use_cases) if use_cases else "N/A"
+
+    # Platforms
+    platforms = modality_obj['metadata'].get('platforms', [])
+    platform_str = ", ".join(platforms) if platforms else "N/A"
+
+    # Example models
+    models = modality_obj['metadata'].get('exampleModels', [])
+    model_str = ", ".join(models) if models else "N/A"
+
+    html = f"""
+    <div class="modality-card">
+        <div class="modality-header">
+            {modality_obj['name']}
+            <span class="{badge_class}">{maturity}</span>
+        </div>
+
+        <div class="modality-meta">
+            <p><strong>🔹 Input</strong><br>{input_str}</p>
+            <p><strong>🔸 Output</strong><br>**Primary:** {output_primary}{audio_info}</p>
+        </div>
+
+        <details>
+            <summary><strong>📊 Characteristics</strong></summary>
+            <div style="margin: 10px; padding: 10px; background: #fafafa; border-radius: 5px;">
+                {char_str}
+            </div>
+        </details>
+
+        <details>
+            <summary><strong>💡 Common Use Cases</strong></summary>
+            <div style="margin: 10px; padding: 10px; background: #fafafa; border-radius: 5px;">
+                {use_case_str}
+            </div>
+        </details>
+
+        <details>
+            <summary><strong>🛠️ Platforms & Models</strong></summary>
+            <div style="margin: 10px; padding: 10px; background: #fafafa; border-radius: 5px;">
+                <p><strong>Platforms:</strong> {platform_str}</p>
+                <p><strong>Example Models:</strong> {model_str}</p>
+            </div>
+        </details>
+    </div>
+    """
+    return html
+
+def create_overview_page():
+    """Create the main overview/index page"""
+
+    stats_html = "<div style='display: flex; flex-wrap: wrap; justify-content: space-around;'>"
+
+    total_modalities = 0
+    for modality_key, operations in taxonomy_data.items():
+        info = MODALITY_INFO.get(modality_key, {"name": modality_key, "emoji": "📦", "color": "#666"})
+
+        creation_count = len(operations.get('creation', {}).get('modalities', []))
+        editing_count = len(operations.get('editing', {}).get('modalities', []))
+        total_count = creation_count + editing_count
+        total_modalities += total_count
+
+        stats_html += f"""
+        <div class="stat-box" style="border-left: 4px solid {info['color']};">
+            <div style="font-size: 2em;">{info['emoji']}</div>
+            <div style="font-size: 1.2em; font-weight: bold; margin: 10px 0;">{info['name']}</div>
+            <div style="font-size: 0.9em; color: #666;">
+                Creation: {creation_count} | Editing: {editing_count}
+            </div>
+            <div style="font-size: 1.5em; font-weight: bold; color: {info['color']}; margin-top: 10px;">
+                {total_count} modalities
+            </div>
+        </div>
+        """
+
+    stats_html += "</div>"
+
+    overview_html = f"""
+    <div style="text-align: center; padding: 30px;">
+        <h1>🎯 Multimodal AI Taxonomy</h1>
+        <p style="font-size: 1.2em; color: #666; max-width: 800px; margin: 20px auto;">
+            A comprehensive taxonomy for multimodal generative AI capabilities, organized by output modality and operation type.
+        </p>
+        <div style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; padding: 20px; border-radius: 15px; margin: 20px auto; max-width: 300px;">
+            <div style="font-size: 3em; font-weight: bold;">{total_modalities}</div>
+            <div style="font-size: 1.2em;">Total Modalities</div>
+        </div>
+    </div>
+
+    {stats_html}
+
+    <div style="margin: 30px; padding: 20px; background: #f0f7ff; border-radius: 10px; border-left: 4px solid #2196F3;">
+        <h3>📖 How to Use This Space</h3>
+        <p>Navigate through the tabs above to explore different output modalities (Video, Audio, Image, Text, 3D).</p>
+        <p>Each modality is organized into <strong>Creation</strong> (generating new content) and <strong>Editing</strong> (modifying existing content) operations.</p>
+        <p>Click on the details sections to expand and see characteristics, use cases, platforms, and example models.</p>
+    </div>
+    """
+
+    return overview_html
+
+def create_modality_page(modality_key, operation_type):
+    """Create a page for a specific modality and operation type"""
+
+    if modality_key not in taxonomy_data:
+        return f"<p>No data found for {modality_key}</p>"
+
+    if operation_type not in taxonomy_data[modality_key]:
+        return f"<p>No {operation_type} data found for {modality_key}</p>"
+
+    data = taxonomy_data[modality_key][operation_type]
+    modalities = data.get('modalities', [])
+
+    info = MODALITY_INFO.get(modality_key, {"name": modality_key, "emoji": "📦", "color": "#666"})
+
+    html = f"""
+    <div style="text-align: center; padding: 20px; background: linear-gradient(135deg, {info['color']}22 0%, {info['color']}44 100%); border-radius: 15px; margin-bottom: 20px;">
+        <h2>{info['emoji']} {info['name']} - {operation_type.title()}</h2>
+        <p style="color: #666;">{data.get('description', '')}</p>
+        <div style="font-size: 1.5em; font-weight: bold; color: {info['color']}; margin-top: 10px;">
+            {len(modalities)} modalities
+        </div>
+    </div>
+    """
+
+    for modality in modalities:
+        html += create_modality_card(modality)
+
+    return html
+
+def create_comparison_table(modality_key):
+    """Create a comparison table for creation vs editing"""
+
+    if modality_key not in taxonomy_data:
+        return pd.DataFrame()
+
+    rows = []
+    for operation_type in ['creation', 'editing']:
+        if operation_type in taxonomy_data[modality_key]:
+            modalities = taxonomy_data[modality_key][operation_type].get('modalities', [])
+            for mod in modalities:
+                rows.append({
+                    'Operation': operation_type.title(),
+                    'Name': mod['name'],
+                    'Primary Input': mod['input']['primary'],
+                    'Primary Output': mod['output']['primary'],
+                    'Maturity': mod['metadata']['maturityLevel'],
+                    'Platforms': len(mod['metadata'].get('platforms', [])),
+                })
+
+    return pd.DataFrame(rows)
+
+# Create the Gradio interface
+with gr.Blocks(css=custom_css, theme=gr.themes.Soft()) as demo:
+
+    gr.Markdown("# 🎯 Multimodal AI Taxonomy Explorer")
+
+    with gr.Tabs():
+        # Overview tab
+        with gr.Tab("🏠 Overview"):
+            gr.HTML(create_overview_page())
+
+        # Video Generation
+        with gr.Tab("🎬 Video"):
+            with gr.Tabs():
+                with gr.Tab("Creation"):
+                    gr.HTML(create_modality_page("video_generation", "creation"))
+                with gr.Tab("Editing"):
+                    gr.HTML(create_modality_page("video_generation", "editing"))
+                with gr.Tab("Comparison"):
+                    gr.Dataframe(create_comparison_table("video_generation"), wrap=True)
+
+        # Audio Generation
+        with gr.Tab("🎵 Audio"):
+            with gr.Tabs():
+                with gr.Tab("Creation"):
+                    gr.HTML(create_modality_page("audio_generation", "creation"))
+                with gr.Tab("Editing"):
+                    gr.HTML(create_modality_page("audio_generation", "editing"))
+                with gr.Tab("Comparison"):
+                    gr.Dataframe(create_comparison_table("audio_generation"), wrap=True)
+
+        # Image Generation
+        with gr.Tab("🖼️ Image"):
+            with gr.Tabs():
+                with gr.Tab("Creation"):
+                    gr.HTML(create_modality_page("image_generation", "creation"))
+                with gr.Tab("Editing"):
+                    gr.HTML(create_modality_page("image_generation", "editing"))
+                with gr.Tab("Comparison"):
+                    gr.Dataframe(create_comparison_table("image_generation"), wrap=True)
+
+        # Text Generation
+        with gr.Tab("📝 Text"):
+            with gr.Tabs():
+                with gr.Tab("Creation"):
+                    gr.HTML(create_modality_page("text_generation", "creation"))
+                with gr.Tab("Editing"):
+                    gr.HTML(create_modality_page("text_generation", "editing"))
+                with gr.Tab("Comparison"):
+                    gr.Dataframe(create_comparison_table("text_generation"), wrap=True)
+
+        # 3D Generation
+        with gr.Tab("🎨 3D"):
+            with gr.Tabs():
+                with gr.Tab("Creation"):
+                    gr.HTML(create_modality_page("3d_generation", "creation"))
+                with gr.Tab("Editing"):
+                    gr.HTML(create_modality_page("3d_generation", "editing"))
+                with gr.Tab("Comparison"):
+                    gr.Dataframe(create_comparison_table("3d_generation"), wrap=True)
+
+        # About tab
+        with gr.Tab("ℹ️ About"):
+            gr.Markdown("""
+            ## About This Taxonomy
+
+            This taxonomy provides a structured classification of multimodal AI capabilities, organized by:
+
+            - **Output Modality**: The primary type of content being generated (video, audio, image, text, 3D)
+            - **Operation Type**: Whether the task involves creation (from scratch) or editing (modifying existing content)
+
+            ### Key Features
+
+            - **Comprehensive Coverage**: Covers all major multimodal AI capabilities
+            - **Structured Metadata**: Each modality includes input/output specs, characteristics, maturity level, use cases, platforms, and example models
+            - **Fine-grained Classification**: Goes beyond simple input/output categorization to capture nuanced differences
+
+            ### Data Schema
+
+            Each modality entry includes:
+            - Unique identifier and human-readable name
+            - Input specifications (primary and secondary modalities)
+            - Output specifications (with audio metadata for video outputs)
+            - Characteristics (process type, audio handling, motion type, etc.)
+            - Metadata (maturity level, use cases, platforms, example models)
+
+            ### Dataset
+
+            This visualization is powered by the [multimodal-ai-taxonomy](https://huggingface.co/datasets/danielrosehill/multimodal-ai-taxonomy) dataset on Hugging Face.
+
+            ### Maturity Levels
+
+            - **Mature**: Well-established, widely available, production-ready
+            - **Emerging**: Growing adoption, increasingly stable
+            - **Experimental**: Cutting-edge, limited availability, proof-of-concept
+            """)
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,3 @@
+gradio==5.49.1
+datasets
+pandas