| import os
|
| from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
|
|
|
| MODEL_LIST = {
|
| "gpt2": {"path": "openai-community/gpt2", "save_dir": "./models/gpt2"},
|
| "gpt2-medium": {"path": "openai-community/gpt2-medium", "save_dir": "./models/gpt2-medium"},
|
| "gpt2-persian": {"path": "flax-community/gpt2-medium-persian", "save_dir": "./models/gpt2-medium-persian"},
|
| "gpt2-large": {"path": "openai-community/gpt2-large", "save_dir": "./models/gpt2-large"},
|
| "codegen": {"path": "Salesforce/codegen-350M-mono", "save_dir": "./models/codegen"},
|
| "dialogpt": {"path": "microsoft/DialoGPT-small", "save_dir": "./models/dialogpt"},
|
| "dialogpt-medium": {"path": "microsoft/DialoGPT-medium", "save_dir": "./models/dialogpt-medium"},
|
| "dialogpt-large": {"path": "microsoft/DialoGPT-large", "save_dir": "./models/dialogpt-large"}
|
| }
|
|
|
| def download_and_save_models():
|
| """
|
| دانلود و ذخیره تمام مدلها در مسیرهای مشخصشده.
|
| """
|
| for model_name, model_info in MODEL_LIST.items():
|
| model_path = model_info["path"]
|
| save_dir = model_info["save_dir"]
|
|
|
| print(f"Downloading and saving model: {model_name} to folder: {save_dir}")
|
|
|
| if not os.path.exists(save_dir):
|
| os.makedirs(save_dir, exist_ok=True)
|
|
|
|
|
| model = AutoModelForCausalLM.from_pretrained(model_path)
|
| tokenizer = AutoTokenizer.from_pretrained(model_path)
|
| model.save_pretrained(save_dir)
|
| tokenizer.save_pretrained(save_dir)
|
|
|
| print(f"Model {model_name} saved to {save_dir}")
|
| else:
|
| print(f"Model {model_name} already exists in {save_dir}")
|
|
|
| if __name__ == "__main__":
|
| download_and_save_models()
|
|
|
