app.py

import gradio as gr
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.repocard import metadata_load
import requests
import re
import pandas as pd
from huggingface_hub import ModelCard

def pass_emoji(passed):
    if passed is True:
        passed = "✅"
    else:
        passed = "❌"
    return passed

api = HfApi()

def get_user_audio_classification_models(hf_username):
    """
    List the user's Audio Classification models
    :param hf_username: User HF username
    """

    models = api.list_models(author=hf_username, filter=["audio-classification"])
    user_model_ids = [x.modelId for x in models]
    models_gtzan = []

    for model in user_model_ids:
        meta = get_metadata(model)
        if meta is None:
            continue
        if meta["datasets"] == ['marsyas/gtzan']:
            models_gtzan.append(model)
    
    return models_gtzan


def get_metadata(model_id):
  """
  Get model metadata (contains evaluation data)
  :param model_id
  """
  try:
    readme_path = hf_hub_download(model_id, filename="README.md")
    return metadata_load(readme_path)
  except requests.exceptions.HTTPError:
    # 404 README.md not found
    return None


def extract_accuracy(model_card_content):
    """
    Extract the accuracy value from the models' model card
    :param model_card_content: model card content
    """
    accuracy_pattern = r"Accuracy: (\d+\.\d+)"
    match = re.search(accuracy_pattern, model_card_content)
    if match:
        accuracy = match.group(1)
        return float(accuracy)
    else:
        return None


def parse_metrics_accuracy(model_id):
  """
  Get model card and parse it
  :param model_id: model id
  """
  card = ModelCard.load(model_id)
  return extract_accuracy(card.content)

  
def calculate_best_acc_result(user_model_ids):
  """
  Calculate the best results of a unit
  :param user_model_ids: RL models of a user
  """

  best_result = -100
  best_model = ""

  for model in user_model_ids:
    meta = get_metadata(model)
    if meta is None:
      continue
    accuracy = parse_metrics_accuracy(model)
    if accuracy > best_result:
      best_result = accuracy
      best_model = meta['model-index'][0]["name"]
      
  return best_result, best_model


def certification(hf_username):
  results_certification = [
      {
          "unit": "Unit 4: Audio Classification",
          "task": "audio-classification",
          "baseline_metric": 0.87,
          "best_result": 0,
          "best_model_id": "",
          "passed_": False
      },
  {
          "unit": "Unit 5: TBD",
          "task": "TBD",          
          "baseline_metric": 0.99,
          "best_result": 0,
          "best_model_id": "",
          "passed_": False
  },
  {
          "unit": "Unit 6: TBD",
          "task": "TBD",
          "baseline_metric": 0.99,
          "best_result": 0,
          "best_model_id": "",
          "passed_": False
  },
  {
          "unit": "Unit 7: TBD",
          "task": "TBD",
          "baseline_metric": 0.99,
          "best_result": 0,
          "best_model_id": "",
          "passed_": False
  },
  ] 

  for unit in results_certification:
    if unit["task"] == "audio-classification":
      user_models = get_user_audio_classification_models(hf_username)
      best_result, best_model_id = calculate_best_acc_result(user_models)
      unit["best_result"] = best_result
      unit["best_model_id"] = best_model_id
      if unit["best_result"] >= unit["baseline_metric"]: 
        unit["passed_"] = True
        unit["passed"] = pass_emoji(unit["passed_"])
    else: 
      # TBD for other units
      unit["passed"] = pass_emoji(unit["passed_"])
      continue        
    
  print(results_certification)
 
  df = pd.DataFrame(results_certification)
  df = df[['passed', 'unit', 'task', 'baseline_metric', 'best_result', 'best_model_id']]
  return df


with gr.Blocks() as demo:
    gr.Markdown(f"""
    # 🏆 Check your progress in the Audio Course 🏆
    You can check your progress here.
    
    - To get a certificate of completion, you must **pass 3 out of 4 assignments before July 31st 2023**.
    - To get an honors certificate, you must **pass 4 out of 4 assignments before July 31st 2023**.

    To pass an assignment, your model's metric should be equal or higher than the baseline metric

    **When min_result = -100 it means that you just need to push a model to pass this hands-on.**
    
    Just type your Hugging Face Username 🤗 (in my case MariaK)
    """)
    
    hf_username = gr.Textbox(placeholder="MariaK", label="Your Hugging Face Username")
    check_progress_button = gr.Button(value="Check my progress")
    output = gr.components.Dataframe(value=certification(hf_username))    
    check_progress_button.click(fn=certification, inputs=hf_username, outputs=output)

demo.launch()