update

app.py

@@ -1,47 +1,169 @@
 import pandas as pd
 import gradio as gr
+import requests
+import yaml
+import os
+import re
+import asyncio
+import aiohttp
+import pandas as pd
+from tqdm import tqdm
 
-# Load the CSV file
-model_test_results = pd.read_csv('test_results_by_type.csv')
-# Get models with failed tests and their failure counts
-failed_models_counts = model_test_results[
-    (model_test_results['test_type'] == 'failed') & 
-    (model_test_results['number_of_tests'] > 0)
-].groupby('model')['number_of_tests'].first().to_dict()
 
-# Add ❌ and failure count to model names that have failures, ✅ for passing models
-model_test_results['model'] = model_test_results.apply(
-    lambda row: f"{row['model']} ❌ ({failed_models_counts[row['model']]})" if row['model'] in failed_models_counts else f"{row['model']} ✅",
-    axis=1
-)
+def get_audio_models():
+    url = "https://raw.githubusercontent.com/huggingface/transformers/main/docs/source/en/_toctree.yml"
+    response = requests.get(url)
+    
+    if response.status_code != 200:
+        print("Failed to fetch the YAML file")
+        return []
 
-# Separate failed tests and other tests
-failed_tests = model_test_results[model_test_results['test_type'] == 'failed'].sort_values('number_of_tests', ascending=False)
-other_tests = model_test_results[model_test_results['test_type'] != 'failed']
+    toctree_content = yaml.safe_load(response.text)
+    
+    for section in toctree_content:
+        if section.get('title') == 'API':
+            for subsection in section.get('sections', []):
+                if subsection.get('title') == 'Models':
+                    for model_section in subsection.get('sections', []):
+                        if model_section.get('title') == 'Audio models':
+                            return [audio_model.get('local').split('/')[-1].lower().replace('-', '_') for audio_model in model_section.get('sections', []) if 'local' in audio_model]
+    
+    return []
 
-# Concatenate the dataframes
-model_test_results = pd.concat([failed_tests, other_tests])
 
+def fetch_and_process_ci_results(job_id):
+    github_token = os.environ.get('GITHUB_TOKEN')
+    if not github_token:
+        raise ValueError("GitHub token not found in environment variables")
+
+    headers = {
+        "Authorization": f"token {github_token}",
+        "Accept": "application/vnd.github+json"
+    }
+
+    audio_models = get_audio_models()
+    non_tested_models = [
+        "xls_r",
+        "speech_to_text_2",
+        "mctct", 
+        "xlsr_wav2vec2",
+        "mms"
+    ]
+
+    url = f"https://api.github.com/repos/huggingface/transformers/actions/runs/{job_id}/jobs"
+
+    audio_model_jobs = {audio_model: [] for audio_model in audio_models}
+
+    def process_jobs(jobs_data):
+        for job in jobs_data['jobs']:
+            if "Model CI" in job['name'] and "models" in job['name']:
+                match = re.search(r'models/([^/)]+)', job['name'])
+                if match:
+                    model_name = match.group(1).lower()
+                    if model_name in audio_model_jobs:
+                        audio_model_jobs[model_name].append(job['id'])
+
+    async def fetch_and_process_jobs(session, url):
+        async with session.get(url, headers=headers) as response:
+            jobs_data = await response.json()
+            process_jobs(jobs_data)
+            return response.links.get('next', {}).get('url')
+
+    async def fetch_all_jobs():
+        async with aiohttp.ClientSession() as session:
+            next_url = url
+            with tqdm(desc="Fetching jobs", unit="page") as pbar:
+                while next_url:
+                    next_url = await fetch_and_process_jobs(session, next_url)
+                    pbar.update(1)
+
+    def parse_test_results(text):
+        pattern = r'=+ (?:(\d+) failed,?\s*)?(?:(\d+) passed,?\s*)?(?:(\d+) skipped,?\s*)?(?:\d+ warnings?\s*)?in \d+\.\d+s'
+        match = re.search(pattern, text)
+        if match:
+            failed = int(match.group(1)) if match.group(1) else 0
+            passed = int(match.group(2)) if match.group(2) else 0
+            skipped = int(match.group(3)) if match.group(3) else 0
+            return {'failed': failed, 'passed': passed, 'skipped': skipped}
+        raise Exception("Could not find test summary in logs")
+
+    def retrieve_job_logs(job_id, job_name):
+        url = f"https://api.github.com/repos/huggingface/transformers/actions/jobs/{job_id}"
+        response = requests.get(url, headers=headers)
+        logs_url = f"https://api.github.com/repos/huggingface/transformers/actions/jobs/{job_id}/logs"
+        logs_response = requests.get(logs_url, headers=headers)
+        logs = logs_response.text
+        test_summary = parse_test_results(logs)
+        test_summary["model"] = job_name
+        test_summary["conclusion"] = response.json()['conclusion']
+        return test_summary
+
+    # Fetch initial jobs and run asynchronous job fetching
+    response = requests.get(url, headers=headers)
+    jobs = response.json()
+    process_jobs(jobs)
+    asyncio.run(fetch_all_jobs())
+
+    # Retrieve job logs and process results
+    results = []
+    for job_name, job_ids in tqdm(audio_model_jobs.items()):
+        for job_id in job_ids:
+            result = retrieve_job_logs(job_id, job_name)
+            results.append(result)
+
+    # Process results into DataFrame and save to CSV
+    df = (pd.DataFrame(results)
+                .melt(id_vars=['model', 'conclusion'],
+                    value_vars=['failed', 'passed', 'skipped'], 
+                    var_name='test_type',
+                    value_name='number_of_tests')
+                .groupby(['model', 'conclusion', 'test_type'])
+                .agg({'number_of_tests': 'sum'})
+                .reset_index())
+
+    df.to_csv('test_results_by_type.csv', index=False)
+
+
+def load_and_process_data():
+    # Load the CSV file
+    model_test_results = pd.read_csv('test_results_by_type.csv')
+    # Get models with failed tests and their failure counts
+    failed_models_counts = model_test_results[
+        (model_test_results['test_type'] == 'failed') & 
+        (model_test_results['number_of_tests'] > 0)
+    ].groupby('model')['number_of_tests'].first().to_dict()
+
+    # Add ❌ and failure count to model names that have failures, ✅ for passing models
+    model_test_results['model'] = model_test_results.apply(
+        lambda row: f"{row['model']} ❌ ({failed_models_counts[row['model']]})" if row['model'] in failed_models_counts else f"{row['model']} ✅",
+        axis=1
+    )
+
+    # Separate failed tests and other tests
+    failed_tests = model_test_results[model_test_results['test_type'] == 'failed'].sort_values('number_of_tests', ascending=False)
+    other_tests = model_test_results[model_test_results['test_type'] != 'failed']
+
+    # Concatenate the dataframes
+    model_test_results = pd.concat([failed_tests, other_tests])
 
-# Create the Gradio interface
-with gr.Blocks() as test_results_viz:
-    gr.Markdown("# Test Results by Model")
-    
     # Sort models by success/failure and number of failed tests
     model_order = model_test_results.sort_values(
         by=['conclusion', 'test_type', 'number_of_tests'],
         ascending=[True, False, False]
     )['model'].unique().tolist()
-    
-    # Create the stacked bar plot using Gradio's BarPlot
-    test_results_plot = gr.BarPlot(
+
+    return model_test_results, model_order, failed_models_counts
+
+
+def create_bar_plot(model_test_results, model_order, failed_models_counts):
+    return gr.BarPlot(
         model_test_results,
         x="model",
         y="number_of_tests",  # Base layer
         color="test_type",  # Color by pass/fail status
         color_map={"passed": "#008550", "skipped": "#F0B702", "failed": "#8B1710"},
         title="Test Results by Model",
-        x_title="Model", 
+        x_title=f"Models ({len(failed_models_counts)} failing / {len(model_order)} total)", 
         y_title="Number of Tests",
         height=600,
         width=1000,
@@ -49,5 +171,48 @@ with gr.Blocks() as test_results_viz:
         x_order=model_order  # Set custom order of x-axis
     )
 
+
+# Create the Gradio interface
+with gr.Blocks() as results_viz:
+    gr.Markdown("# Test Results by Model")
+    
+    model_test_results, model_order, failed_models_counts = load_and_process_data()
+    test_results_plot = create_bar_plot(model_test_results, model_order, failed_models_counts)
+    
+    with gr.Row():
+        refresh_btn = gr.Button(
+            value="Refresh CI Results (~2mn)",
+            variant="primary"
+        )
+        refresh_status = gr.Textbox()
+
+    def check_and_refresh():
+        # For now just return hardcoded ID
+        latest_ci_id = "15549432276"
+        
+        try:
+            with open("ci_id.txt", "r") as f:
+                current_ci_id = f.read().strip()
+        except FileNotFoundError:
+            current_ci_id = ""
+            
+        if latest_ci_id == current_ci_id:
+            return "No new CI results available yet.", test_results_plot
+        else:
+            fetch_and_process_ci_results(latest_ci_id)
+            with open("ci_id.txt", "w") as f:
+                f.write(latest_ci_id)
+            
+            # Reload and reprocess the data
+            new_model_test_results, new_model_order, new_failed_models_counts = load_and_process_data()
+            
+            # Create a new BarPlot with the updated data
+            new_test_results_plot = create_bar_plot(new_model_test_results, new_model_order, new_failed_models_counts)
+            
+            return "CI results refreshed successfully!", new_test_results_plot
+                
+    refresh_btn.click(fn=check_and_refresh, outputs=[refresh_status, test_results_plot])
+
+
 if __name__ == "__main__":
-    test_results_viz.launch()
+    results_viz.launch()

ci_id.txt

@@ -0,0 +1 @@
+15549432276

fetch_ci_results.py

@@ -0,0 +1,120 @@
+import requests
+import yaml
+import os
+import re
+import asyncio
+import aiohttp
+import pandas as pd
+from tqdm import tqdm
+
+def get_audio_models():
+    url = "https://raw.githubusercontent.com/huggingface/transformers/main/docs/source/en/_toctree.yml"
+    response = requests.get(url)
+    
+    if response.status_code != 200:
+        print("Failed to fetch the YAML file")
+        return []
+
+    toctree_content = yaml.safe_load(response.text)
+    
+    for section in toctree_content:
+        if section.get('title') == 'API':
+            for subsection in section.get('sections', []):
+                if subsection.get('title') == 'Models':
+                    for model_section in subsection.get('sections', []):
+                        if model_section.get('title') == 'Audio models':
+                            return [audio_model.get('local').split('/')[-1].lower().replace('-', '_') for audio_model in model_section.get('sections', []) if 'local' in audio_model]
+    
+    return []
+
+def fetch_and_process_ci_results(job_id):
+    github_token = os.environ.get('GITHUB_TOKEN')
+    if not github_token:
+        raise ValueError("GitHub token not found in environment variables")
+
+    headers = {
+        "Authorization": f"token {github_token}",
+        "Accept": "application/vnd.github+json"
+    }
+
+    audio_models = get_audio_models()
+    non_tested_models = [
+        "xls_r",
+        "speech_to_text_2",
+        "mctct", 
+        "xlsr_wav2vec2",
+        "mms"
+    ]
+
+    url = f"https://api.github.com/repos/huggingface/transformers/actions/runs/{job_id}/jobs"
+
+    audio_model_jobs = {audio_model: [] for audio_model in audio_models}
+
+    def process_jobs(jobs_data):
+        for job in jobs_data['jobs']:
+            if "Model CI" in job['name'] and "models" in job['name']:
+                match = re.search(r'models/([^/)]+)', job['name'])
+                if match:
+                    model_name = match.group(1).lower()
+                    if model_name in audio_model_jobs:
+                        audio_model_jobs[model_name].append(job['id'])
+
+    async def fetch_and_process_jobs(session, url):
+        async with session.get(url, headers=headers) as response:
+            jobs_data = await response.json()
+            process_jobs(jobs_data)
+            return response.links.get('next', {}).get('url')
+
+    async def fetch_all_jobs():
+        async with aiohttp.ClientSession() as session:
+            next_url = url
+            with tqdm(desc="Fetching jobs", unit="page") as pbar:
+                while next_url:
+                    next_url = await fetch_and_process_jobs(session, next_url)
+                    pbar.update(1)
+
+    def parse_test_results(text):
+        pattern = r'=+ (?:(\d+) failed,?\s*)?(?:(\d+) passed,?\s*)?(?:(\d+) skipped,?\s*)?(?:\d+ warnings?\s*)?in \d+\.\d+s'
+        match = re.search(pattern, text)
+        if match:
+            failed = int(match.group(1)) if match.group(1) else 0
+            passed = int(match.group(2)) if match.group(2) else 0
+            skipped = int(match.group(3)) if match.group(3) else 0
+            return {'failed': failed, 'passed': passed, 'skipped': skipped}
+        raise Exception("Could not find test summary in logs")
+
+    def retrieve_job_logs(job_id, job_name):
+        url = f"https://api.github.com/repos/huggingface/transformers/actions/jobs/{job_id}"
+        response = requests.get(url, headers=headers)
+        logs_url = f"https://api.github.com/repos/huggingface/transformers/actions/jobs/{job_id}/logs"
+        logs_response = requests.get(logs_url, headers=headers)
+        logs = logs_response.text
+        test_summary = parse_test_results(logs)
+        test_summary["model"] = job_name
+        test_summary["conclusion"] = response.json()['conclusion']
+        return test_summary
+
+    # Fetch initial jobs and run asynchronous job fetching
+    response = requests.get(url, headers=headers)
+    jobs = response.json()
+    process_jobs(jobs)
+    asyncio.run(fetch_all_jobs())
+
+    # Retrieve job logs and process results
+    results = []
+    for job_name, job_ids in tqdm(audio_model_jobs.items()):
+        for job_id in job_ids:
+            result = retrieve_job_logs(job_id, job_name)
+            results.append(result)
+
+    # Process results into DataFrame and save to CSV
+    df = (pd.DataFrame(results)
+                .melt(id_vars=['model', 'conclusion'],
+                    value_vars=['failed', 'passed', 'skipped'], 
+                    var_name='test_type',
+                    value_name='number_of_tests')
+                .groupby(['model', 'conclusion', 'test_type'])
+                .agg({'number_of_tests': 'sum'})
+                .reset_index())
+
+    df.to_csv('test_results_by_type.csv', index=False)

test_results_by_type.csv

@@ -1,58 +1,61 @@
 model,conclusion,test_type,number_of_tests
 audio_spectrogram_transformer,success,failed,0
-audio_spectrogram_transformer,success,passed,242
-audio_spectrogram_transformer,success,skipped,184
+audio_spectrogram_transformer,success,passed,234
+audio_spectrogram_transformer,success,skipped,192
 bark,failure,failed,4
-bark,failure,passed,442
+bark,failure,passed,438
 bark,failure,skipped,452
 clap,success,failed,0
-clap,success,passed,360
-clap,success,skipped,522
+clap,success,passed,356
+clap,success,skipped,526
+csm,failure,failed,1
+csm,failure,passed,109
+csm,failure,skipped,91
 csm,success,failed,0
-csm,success,passed,224
-csm,success,skipped,180
-dac,failure,failed,8
-dac,failure,passed,120
-dac,failure,skipped,270
+csm,success,passed,112
+csm,success,skipped,89
+dac,failure,failed,22
+dac,failure,passed,105
+dac,failure,skipped,271
 encodec,failure,failed,2
-encodec,failure,passed,126
-encodec,failure,skipped,266
-fastspeech2_conformer,failure,failed,6
-fastspeech2_conformer,failure,passed,322
+encodec,failure,passed,122
+encodec,failure,skipped,270
+fastspeech2_conformer,failure,failed,5
+fastspeech2_conformer,failure,passed,323
 fastspeech2_conformer,failure,skipped,360
 granite_speech,success,failed,0
-granite_speech,success,passed,245
+granite_speech,success,passed,243
 granite_speech,success,skipped,89
 hubert,failure,failed,2
-hubert,failure,passed,466
-hubert,failure,skipped,416
+hubert,failure,passed,338
+hubert,failure,skipped,544
 mimi,failure,failed,1
 mimi,failure,passed,68
 mimi,failure,skipped,49
 mimi,success,failed,0
 mimi,success,passed,67
 mimi,success,skipped,51
-moonshine,failure,failed,9
+moonshine,failure,failed,5
 moonshine,failure,passed,157
-moonshine,failure,skipped,188
-moshi,failure,failed,11
-moshi,failure,passed,465
+moonshine,failure,skipped,192
+moshi,failure,failed,10
+moshi,failure,passed,462
 moshi,failure,skipped,486
 musicgen,success,failed,0
-musicgen,success,passed,482
+musicgen,success,passed,478
 musicgen,success,skipped,410
 musicgen_melody,failure,failed,2
-musicgen_melody,failure,passed,260
-musicgen_melody,failure,skipped,148
+musicgen_melody,failure,passed,256
+musicgen_melody,failure,skipped,150
 musicgen_melody,success,failed,0
-musicgen_melody,success,passed,256
-musicgen_melody,success,skipped,154
+musicgen_melody,success,passed,252
+musicgen_melody,success,skipped,156
 pop2piano,success,failed,0
 pop2piano,success,passed,134
 pop2piano,success,skipped,316
 seamless_m4t,success,failed,0
-seamless_m4t,success,passed,454
-seamless_m4t,success,skipped,428
+seamless_m4t,success,passed,450
+seamless_m4t,success,skipped,432
 seamless_m4t_v2,success,failed,0
 seamless_m4t_v2,success,passed,192
 seamless_m4t_v2,success,skipped,296
@@ -62,12 +65,15 @@ sew,success,skipped,198
 sew_d,success,failed,0
 sew_d,success,passed,100
 sew_d,success,skipped,258
-speech_to_text,failure,failed,6
-speech_to_text,failure,passed,535
-speech_to_text,failure,skipped,425
+speech_to_text,failure,failed,2
+speech_to_text,failure,passed,228
+speech_to_text,failure,skipped,252
+speech_to_text,success,failed,0
+speech_to_text,success,passed,227
+speech_to_text,success,skipped,255
 speecht5,success,failed,0
-speecht5,success,passed,759
-speecht5,success,skipped,915
+speecht5,success,passed,755
+speecht5,success,skipped,917
 unispeech,success,failed,0
 unispeech,success,passed,174
 unispeech,success,skipped,190
@@ -75,14 +81,14 @@ unispeech_sat,success,failed,0
 unispeech_sat,success,passed,326
 unispeech_sat,success,skipped,292
 univnet,success,failed,0
-univnet,success,passed,127
-univnet,success,skipped,163
+univnet,success,passed,123
+univnet,success,skipped,167
 vits,failure,failed,2
 vits,failure,passed,248
 vits,failure,skipped,304
 wav2vec2,success,failed,0
-wav2vec2,success,passed,828
-wav2vec2,success,skipped,608
+wav2vec2,success,passed,684
+wav2vec2,success,skipped,752
 wav2vec2_bert,success,failed,0
 wav2vec2_bert,success,passed,162
 wav2vec2_bert,success,skipped,334
@@ -96,5 +102,5 @@ wavlm,success,failed,0
 wavlm,success,passed,122
 wavlm,success,skipped,242
 whisper,failure,failed,4
-whisper,failure,passed,1122
-whisper,failure,skipped,648
+whisper,failure,passed,1024
+whisper,failure,skipped,742