| import os | |
| from transformers import AutoTokenizer, AutoImageProcessor | |
| import traceback | |
| MODEL_PATH = "/workspace/ankahi/punjabi_gemma/ankahi" | |
| print("--- Testing Tokenizer ---") | |
| try: | |
| tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH) | |
| print("Tokenizer loaded successfully.") | |
| print(f"Vocab size: {len(tokenizer)}") | |
| except Exception as e: | |
| print(f"Tokenizer failed: {e}") | |
| traceback.print_exc() | |
| print("\n--- Testing Image Processor ---") | |
| try: | |
| # Some models might have image_processor_config.json or preprocessor_config.json | |
| image_processor = AutoImageProcessor.from_pretrained(MODEL_PATH) | |
| print("Image Processor loaded successfully.") | |
| except Exception as e: | |
| print(f"Image Processor failed: {e}") | |
| # traceback.print_exc() # Often fails if not multimodal, that's fine | |