Spaces:
Running
Running
File size: 1,897 Bytes
197f5ec |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
import requests
import json
import pandas as pd
from src.constants import MODEL_REPOSITORY_URL
def clean_models_data(df):
dict_providers = {
'google': 'Google',
'mistralai': 'MistralAI',
'meta-llama': 'Meta',
'openai': 'OpenAI',
'anthropic': 'Anthropic',
'cohere': 'Cohere',
'microsoft': 'Microsoft',
'mistral-community': 'Mistral Community',
'databricks': 'Databricks'
}
df.drop('type', axis=1, inplace=True)
df.loc[df['name'].str.contains('/'), 'name_clean'] = df.loc[df['name'].str.contains('/'), 'name'].str.split('/').str[1]
df['name_clean'] = df['name_clean'].fillna(df['name'])
df['name_clean'] = df['name_clean'].replace({'-': ' '}, regex = True)
df.loc[df['provider'] == 'huggingface_hub', 'provider_clean'] = df.loc[df['provider'] == 'huggingface_hub', 'name'].str.split('/').str[0]
df['provider_clean'] = df['provider_clean'].fillna(df['provider'])
df['provider_clean'] = df['provider_clean'].replace(dict_providers, regex = True)
df['architecture_type'] = df['architecture'].apply(lambda x: x['type'])
df['architecture_parameters'] = df['architecture'].apply(lambda x: x['parameters'])
df['warnings'] = df['warnings'].apply(lambda x: ', '.join(x) if x else None).fillna('none')
df['warning_arch'] = df['warnings'].apply(lambda x: 'model-arch-not-released' in x)
df['warning_multi_modal'] = df['warnings'].apply(lambda x: 'model-arch-multimodal' in x)
return df[['provider', 'provider_clean', 'name', 'name_clean', 'architecture_type', 'architecture_parameters', 'warning_arch', 'warning_multi_modal']]
def load_models():
resp = requests.get(MODEL_REPOSITORY_URL)
data = json.loads(resp.text)
df = pd.DataFrame(data['models'])
return clean_models_data(df)
|