Buckets:
| """Compare each TIPSv2 model against its PR revision. | |
| For each model: | |
| - load with AutoModel (main revision) and run a single forward pass on the first | |
| reference image | |
| - load the same model again from its open PR revision and run the same forward pass | |
| - iterate over the forward outputs and compare the two sets of tensors | |
| - print whether any output differs between the two revisions | |
| All models are loaded with trust_remote_code=True. | |
| """ | |
| import gc | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).parent.parent / "transformers/src")) | |
| import torch | |
| from PIL import Image | |
| from torchvision import transforms | |
| from transformers import AutoModel | |
| if hasattr(torch.backends, "cuda"): | |
| torch.backends.cuda.matmul.allow_tf32 = False | |
| if hasattr(torch.backends.cudnn, "allow_tf32"): | |
| torch.backends.cudnn.allow_tf32 = False | |
| if hasattr(torch.backends.cudnn, "conv") and hasattr(torch.backends.cudnn.conv, "fp32_precision"): | |
| torch.backends.cudnn.conv.fp32_precision = "ieee" | |
| SCRIPT_DIR = Path(__file__).parent | |
| MODELS = [ | |
| "google/tipsv2-b14", | |
| "google/tipsv2-l14", | |
| "google/tipsv2-so400m14", | |
| "google/tipsv2-g14", | |
| "google/tipsv2-b14-dpt", | |
| "google/tipsv2-l14-dpt", | |
| "google/tipsv2-so400m14-dpt", | |
| "google/tipsv2-g14-dpt", | |
| ] | |
| REFERENCE_IMAGE = SCRIPT_DIR / "reference_images/coco_000000039769.jpg" | |
| IMAGE_SIZE = 448 | |
| IMAGE_MEAN = (0, 0, 0) | |
| IMAGE_STD = (1.0, 1.0, 1.0) | |
| def revision_for(model_id: str) -> str: | |
| """Return the open PR revision to compare the main revision against.""" | |
| if model_id.split("/")[-1] == "tipsv2-b14": | |
| return "refs/pr/3" | |
| return "refs/pr/1" | |
| def preprocess_image(path: Path, size: int = IMAGE_SIZE) -> torch.Tensor: | |
| transform = transforms.Compose([ | |
| transforms.Resize((size, size)), | |
| transforms.ToTensor(), | |
| transforms.Normalize(IMAGE_MEAN, IMAGE_STD), | |
| ]) | |
| return transform(Image.open(path).convert("RGB")).unsqueeze(0) | |
| def extract_tensors(outputs) -> dict: | |
| """Return the named tensors of a forward output, regardless of its container type.""" | |
| if hasattr(outputs, "items"): | |
| pairs = outputs.items() | |
| elif hasattr(outputs, "__dict__"): | |
| pairs = vars(outputs).items() | |
| else: | |
| pairs = ((str(index), value) for index, value in enumerate(outputs)) | |
| return { | |
| key: value.detach().cpu() | |
| for key, value in pairs | |
| if isinstance(value, torch.Tensor) | |
| } | |
| def run_forward(model_id: str, revision: str | None, pixel_values: torch.Tensor, device): | |
| model = AutoModel.from_pretrained(model_id, revision=revision, trust_remote_code=True) | |
| model.eval().to(device) | |
| try: | |
| with torch.no_grad(): | |
| outputs = model(pixel_values.to(device)) | |
| # Move every tensor to cpu so it survives the model being freed. | |
| return extract_tensors(outputs) | |
| finally: | |
| del model | |
| gc.collect() | |
| if device.type == "cuda": | |
| torch.cuda.empty_cache() | |
| def main() -> None: | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| print(f"Device: {device}") | |
| print(f"Reference image: {REFERENCE_IMAGE.name}\n") | |
| pixel_values = preprocess_image(REFERENCE_IMAGE) | |
| for model_id in MODELS: | |
| revision = revision_for(model_id) | |
| print(f"=== {model_id} (main vs {revision}) ===") | |
| try: | |
| main_outputs = run_forward(model_id, None, pixel_values, device) | |
| pr_outputs = run_forward(model_id, revision, pixel_values, device) | |
| except Exception as exc: | |
| print(f" ERROR: {exc}\n") | |
| continue | |
| any_different = False | |
| num_keys = len(sorted(set(main_outputs) | set(pr_outputs))) | |
| for key in sorted(set(main_outputs) | set(pr_outputs)): | |
| if key not in main_outputs or key not in pr_outputs: | |
| print(f" {key}: MISSING in one revision") | |
| any_different = True | |
| continue | |
| main_tensor = main_outputs[key] | |
| pr_tensor = pr_outputs[key] | |
| if main_tensor.shape != pr_tensor.shape: | |
| print(f" {key}: SHAPE MISMATCH {tuple(main_tensor.shape)} vs {tuple(pr_tensor.shape)}") | |
| any_different = True | |
| continue | |
| if not torch.equal(main_tensor, pr_tensor): | |
| max_abs_diff = (main_tensor.float() - pr_tensor.float()).abs().max().item() | |
| print(f" {key}: DIFFERENT (max_abs_diff={max_abs_diff:.6g})") | |
| any_different = True | |
| if any_different: | |
| print(" -> outputs DIFFER between revisions\n") | |
| else: | |
| print(f" -> all {num_keys} outputs identical\n") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 4.73 kB
- Xet hash:
- 5099ecd682c40e00d3ff292f5bcef6f3ea2e24a9121e6b6a9dbc05377d03de76
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.