davanstrien HF Staff commited on
Commit
43cc1b2
·
verified ·
1 Parent(s): 22fb4ba

Upload probe.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. probe.py +379 -0
probe.py ADDED
@@ -0,0 +1,379 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # /// script
2
+ # requires-python = ">=3.10"
3
+ # dependencies = [
4
+ # "torch>=2.1",
5
+ # "transformers>=4.40",
6
+ # "accelerate>=0.27",
7
+ # "datasets>=2.18",
8
+ # "pycocotools",
9
+ # "huggingface_hub>=0.23",
10
+ # "Pillow",
11
+ # "numpy",
12
+ # "requests",
13
+ # ]
14
+ # ///
15
+ import os, json, time, argparse, io
16
+ import numpy as np
17
+ import torch
18
+ from PIL import Image
19
+
20
+ def parse_args():
21
+ p = argparse.ArgumentParser()
22
+ p.add_argument("--data_dir", default="/data")
23
+ p.add_argument("--base_model", default="facebook/detr-resnet-50")
24
+ p.add_argument("--hub_id", default="harness-race/opencode-r3")
25
+ p.add_argument("--epochs", type=int, default=25)
26
+ p.add_argument("--batch_size", type=int, default=2)
27
+ p.add_argument("--lr", type=float, default=1e-4)
28
+ p.add_argument("--patience", type=int, default=4)
29
+ p.add_argument("--seed", type=int, default=42)
30
+ p.add_argument("--exp_name", default="opencode-r3-detr")
31
+ p.add_argument("--max_steps", type=int, default=0)
32
+ return p.parse_args()
33
+
34
+ ARGS = parse_args()
35
+
36
+ ID2LABEL = {0:"Photograph",1:"Illustration",2:"Map",3:"Comics/Cartoon",4:"Editorial Cartoon",5:"Headline",6:"Advertisement"}
37
+ LABEL2ID = {v:k for k,v in ID2LABEL.items()}
38
+ N = len(ID2LABEL)
39
+
40
+ import datasets as hfds
41
+ from torch.utils.data import Dataset, DataLoader
42
+ from transformers import DetrImageProcessor, DetrForObjectDetection
43
+ from tqdm import tqdm
44
+ from pycocotools.coco import COCO
45
+ from pycocotools.cocoeval import COCOeval
46
+
47
+ def log(*m):
48
+ print("[%s]" % time.strftime("%H:%M:%S"), *m, flush=True)
49
+
50
+ torch.manual_seed(ARGS.seed)
51
+ np.random.seed(ARGS.seed)
52
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
53
+ log("torch", torch.__version__, "device", device, "cuda_avail", torch.cuda.is_available())
54
+
55
+ data_dir = ARGS.data_dir
56
+ if not os.path.isdir(data_dir):
57
+ log("mount %s not found; downloading dataset" % data_dir)
58
+ data_dir = ARGS.data_dir = "biglam/loc_beyond_words"
59
+
60
+ try:
61
+ ds = hfds.load_dataset(data_dir, split="train")
62
+ ds_val = hfds.load_dataset(data_dir, split="validation")
63
+ except Exception as e:
64
+ log("local load failed (%s); falling back to hub dataset" % e)
65
+ data_dir = "biglam/loc_beyond_words"
66
+ ds = hfds.load_dataset(data_dir, split="train")
67
+ ds_val = hfds.load_dataset(data_dir, split="validation")
68
+
69
+ def get_pil(ex):
70
+ img = ex["image"]
71
+ if isinstance(img, dict):
72
+ img = Image.open(io.BytesIO(img["bytes"])).convert("RGB")
73
+ else:
74
+ img = img.convert("RGB")
75
+ return img
76
+
77
+ def fields(ex):
78
+ objs = ex["objects"]
79
+ o = {}
80
+ for k in ("category_id","bbox","area","image_id","id"):
81
+ v = objs.get(k)
82
+ o[k] = list(v) if isinstance(v, list) else v
83
+ return o
84
+
85
+ # Build torch Dataset returning (pil_img, anns_for_processor, size_wh, row_id)
86
+ class DetrDS(Dataset):
87
+ def __init__(self, examples, with_size=True):
88
+ self.examples = examples
89
+ self.sizes = []
90
+ for ex in examples:
91
+ try:
92
+ w,h = ex["width"], ex["height"]
93
+ except Exception:
94
+ w,h = get_pil(ex).size
95
+ self.sizes.append((w,h))
96
+ def __len__(self):
97
+ return len(self.examples)
98
+ def __getitem__(self, i):
99
+ ex = self.examples[i]
100
+ img = get_pil(ex)
101
+ o = fields(ex)
102
+ bboxes = o.get("bbox") or []
103
+ cats = o.get("category_id")
104
+ if not isinstance(cats, list):
105
+ cats = [0]*len(bboxes)
106
+ anns = [{"bbox":[float(x) for x in bboxes[j]], "category_id":int(cats[j])&0x7fffffff} for j in range(len(bboxes))]
107
+ return img, anns, self.sizes[i], i
108
+
109
+ processor = DetrImageProcessor.from_pretrained(ARGS.base_model,
110
+ do_resize=True, size={"shortest_edge":800, "longest_edge":1333})
111
+
112
+ train_ds = DetrDS(list(ds))
113
+ val_ds = DetrDS(list(ds_val))
114
+ log("train", len(train_ds), "val", len(val_ds))
115
+
116
+ def collate_fn(batch):
117
+ # batch: list of (img, anns, size, idx)
118
+ images = [b[0] for b in batch]
119
+ anns = [b[1] for b in batch]
120
+ sizes = [b[2] for b in batch]
121
+ idx = [b[3] for b in batch]
122
+ return images, anns, sizes, idx
123
+
124
+ train_dl = DataLoader(train_ds, batch_size=ARGS.batch_size, shuffle=True, num_workers=2, collate_fn=collate_fn)
125
+ val_dl = DataLoader(val_ds, batch_size=ARGS.batch_size, shuffle=False, num_workers=2, collate_fn=collate_fn)
126
+
127
+ model = DetrForObjectDetection.from_pretrained(ARGS.base_model, num_labels=N,
128
+ ignore_mismatched_sizes=True, id2label=ID2LABEL, label2id=LABEL2ID)
129
+ model.to(device)
130
+
131
+ # distinct backbone (frozen-ish) vs head lr
132
+ backbone = {n for n,_ in model.backbone.named_parameters() if "layers" in n}
133
+ head_params = [p for n,p in model.named_parameters() if not n.startswith("backbone") or "layers" not in n]
134
+ bb_params = [p for n,p in model.backbone.named_parameters() if "layers" in n]
135
+ optim = torch.optim.AdamW([
136
+ {"params": bb_params, "lr": ARGS.lr/10},
137
+ {"params": head_params, "lr": ARGS.lr},
138
+ ], lr=ARGS.lr, weight_decay=1e-4)
139
+
140
+ def build_coco_gt(examples):
141
+ gt = {"images":[],"annotations":[],"categories":[{"id":k,"name":v} for k,v in ID2LABEL.items()]}
142
+ ann_id = 1
143
+ for i, ex in enumerate(examples):
144
+ o = fields(ex)
145
+ w,h = val_ds.sizes[i]
146
+ gt["images"].append({"id":i,"width":w,"height":h})
147
+ bboxes = o.get("bbox") or []
148
+ cats = o.get("category_id")
149
+ if not isinstance(cats, list):
150
+ cats = [0]*len(bboxes)
151
+ for j in range(len(bboxes)):
152
+ b = [float(x) for x in bboxes[j]]
153
+ gt["annotations"].append({"id":ann_id,"image_id":i,"category_id":int(cats[j])&0x7fffffff,
154
+ "bbox":b,"area":b[2]*b[3],"iscrowd":0}); ann_id+=1
155
+ g = COCO(); g.dataset = gt; g.createIndex()
156
+ return g
157
+
158
+ coco_gt = build_coco_gt(list(ds_val))
159
+
160
+ def evaluate():
161
+ model.eval()
162
+ dets = []
163
+ ann_id = 1
164
+ with torch.no_grad():
165
+ for images, anns, sizes, idx in tqdm(val_dl, desc="eval"):
166
+ enc = processor(images=images, return_tensors="pt")
167
+ pv = enc["pixel_values"].to(device); pm = enc["pixel_mask"].to(device)
168
+ out = model(pixel_values=pv, pixel_mask=pm)
169
+ target_sizes = torch.tensor([val_ds.sizes[ii][::-1] for ii in idx.astype(int)] if hasattr(idx,'astype') else [[val_ds.sizes[ii][1], val_ds.sizes[ii][0]] for ii in idx], device=device)
170
+ # idx is list of ints already
171
+ sz = [[val_ds.sizes[ii][1], val_ds.sizes[ii][0]] for ii in idx]
172
+ results = processor.post_process_object_detection(out, threshold=0.0, target_sizes=sz)
173
+ for ii, res in zip(idx, results):
174
+ boxes = res["boxes"].cpu().numpy() # xmin ymin xmax ymax
175
+ scores = res["scores"].cpu().numpy()
176
+ labels = res["labels"].cpu().numpy()
177
+ for b,s,l in zip(boxes,scores,labels):
178
+ if s <= 0.0: continue
179
+ dets.append({"id":ann_id,"image_id":ii,"category_id":int(l),"bbox":[float(b[0]),float(b[1]),float(b[2]-b[0]),float(b[3]-b[1])],"score":float(s)})
180
+ ann_id += 1
181
+ model.train()
182
+ if len(dets)==0:
183
+ return None
184
+ try:
185
+ preds = coco_gt.loadRes(dets)
186
+ evaluator = COCOeval(coco_gt, preds, iouType="bbox")
187
+ # set areas to all, max dets high
188
+ evaluator.params.maxDets = [100, 300, 1000]
189
+ evaluator.evaluate(); evaluator.accumulate(); evaluator.summarize()
190
+ ap = evaluator.stats # [0]=AP@.5:.95 ... [5]=AP50 ... [6]=AR100
191
+ return {"AP": float(ap[0]), "AP50": float(ap[1]), "AP75": float(ap[2]),
192
+ "AP_s": float(ap[3]), "AP_m": float(ap[4]), "AP_l": float(ap[5]),
193
+ "AR_max100": float(ap[6]), "AR_max1000": float(ap[8])}
194
+ except Exception as e:
195
+ log("COCOeval failed:", e)
196
+ return None
197
+
198
+ # ---------- training ----------
199
+ best = {"AP": -1.0}
200
+ no_improve = 0
201
+ global_step = 0
202
+ run_outputs = {}
203
+ t_start = time.time()
204
+
205
+ log("starting training; epochs", ARGS.epochs, "batch", ARGS.batch_size)
206
+ for epoch in range(1, ARGS.epochs+1):
207
+ model.train()
208
+ ep_loss = 0.0; ep_ce = 0.0; ep_bbox = 0.0; ep_giou = 0.0; nb = 0
209
+ for images, anns, sizes, idx in tqdm(train_dl, desc="epoch %d"%epoch):
210
+ enc = processor(images=images, annotations=anns, return_tensors="pt")
211
+ pv = enc["pixel_values"].to(device); pm = enc["pixel_mask"].to(device)
212
+ labels = [{k: v.to(device) for k,v in l.items()} for l in enc.get("labels")]
213
+ out = model(pixel_values=pv, pixel_mask=pm, labels=labels)
214
+ loss = out.loss
215
+ optim.zero_grad()
216
+ loss.backward()
217
+ optim.step()
218
+ ep_loss += loss.item()
219
+ ep_ce += out.loss_ce.item()
220
+ ep_bbox += out.loss_bbox.item()
221
+ ep_giou += out.loss_giou.item()
222
+ nb += 1
223
+ global_step += 1
224
+ if ARGS.max_steps and global_step >= ARGS.max_steps:
225
+ break
226
+ log("epoch %d loss %.4f ce %.4f bbox %.4f giou %.4f" % (epoch, ep_loss/nb, ep_ce/nb, ep_bbox/nb, ep_giou/nb))
227
+ m = evaluate()
228
+ log("VAL", json.dumps(m))
229
+ if m and m["AP"] > best["AP"]:
230
+ best = m
231
+ best.update({"epoch": epoch, "global_step": global_step})
232
+ no_improve = 0
233
+ torch.save({"state_dict": model.state_dict()}, os.path.join(os.getcwd(), "best_model.pt"))
234
+ log("saved new best AP=%.4f" % m["AP"])
235
+ else:
236
+ no_improve += 1
237
+ if no_improve >= ARGS.patience:
238
+ log("early stop after epoch", epoch)
239
+ break
240
+ if ARGS.max_steps and global_step >= ARGS.max_steps:
241
+ break
242
+ # decay lr slowly
243
+ # (optional) torch.optim.lr_scheduler not added; keep fixed
244
+
245
+ run_outputs["best"] = best
246
+ run_outputs["epochs_run"] = epoch
247
+ run_outputs["elapsed_sec"] = round(time.time()-t_start, 1)
248
+ log("best val result:", best)
249
+
250
+ # ---------- reload best & push ----------
251
+ if os.path.exists("best_model.pt"):
252
+ sd = torch.load("best_model.pt", map_location="cpu")
253
+ model.load_state_dict(sd["state_dict"])
254
+ else:
255
+ best = evaluate() or {}
256
+ run_outputs["best"] = best
257
+
258
+ model.push_to_hub(ARGS.hub_id, safe_serialization=True)
259
+ processor.push_to_hub(ARGS.hub_id)
260
+ log("pushed weights + preprocessor")
261
+
262
+ # build model card
263
+ card = {
264
+ "library_name": "transformers",
265
+ "pipeline_tag": "object-detection",
266
+ "license": "apache-2.0",
267
+ "tags": ["object-detection","detr","computer-vision","document-layout-analysis","pytorch"],
268
+ "base_model": ARGS.base_model,
269
+ "model-index": [{
270
+ "name": ARGS.exp_name,
271
+ "results": [{
272
+ "task": {"type":"object-detection"},
273
+ "dataset": {"type":"biglam/loc_beyond_words","name":"Beyond Words (Testing)","config":"default"},
274
+ "metrics": [
275
+ {"type":"Average Precision","name":"mAP @[IoU=0.50:0.95]","value": round(run_outputs["best"].get("AP",0.0),4)},
276
+ {"type":"Average Precision","name":"mAP @[IoU=0.50]","value": round(run_outputs["best"].get("AP50",0.0),4)},
277
+ ]
278
+ }]
279
+ }]
280
+ }
281
+
282
+
283
+ def build_readme(res):
284
+ m = res if res else {}
285
+ class Row:
286
+ pass
287
+ def f(k, d=.0):
288
+ return "%.4f" % m.get(k, d)
289
+ front = json.dumps(card, indent=2)
290
+ return f"""---
291
+ {front}
292
+ ---
293
+
294
+ # opencode-r3: Beyond Words Object Detection (DETR-ResNet-50)
295
+
296
+ Model for detecting visual content regions in WWI-era historical newspaper pages
297
+ from the US Library of Congress **Beyond Words / Chronicling America** collection.
298
+ The base model `{ARGS.base_model}` (Apache-2.0) was fine-tuned on {len(train_ds)} train images
299
+ and evaluated on {len(val_ds)} validation images over **7 classes**:
300
+ Photograph, Illustration, Map, Comics/Cartoon, Editorial Cartoon, Headline, Advertisement.
301
+
302
+ ## Model Details
303
+
304
+ | | |
305
+ |---|---|
306
+ | **Base model** | {ARGS.base_model} (DETR, ResNet-50 backbone) |
307
+ | **License** | Apache-2.0 (open, shareable) |
308
+ | **Architecture** | `DetrForObjectDetection` |
309
+ | **Task** | Object detection / document layout analysis |
310
+ | **Dataset** | [biglam/loc_beyond_words](https://huggingface.co/datasets/biglam/loc_beyond_words) (CC0-1.0) |
311
+ | **Splits** | {len(train_ds)} train / {len(val_ds)} validation |
312
+
313
+ ## Training Procedure
314
+
315
+ - **Optimizer:** AdamW (head LR {ARGS.lr}, backbone LR {ARGS.lr/10:g})
316
+ - **Epochs:** {run_outputs.get('epochs_run','?')} (early-stopped, patience {ARGS.patience})
317
+ - **Batch size:** {ARGS.batch_size}
318
+ - **Image size:** longest edge 1333 / shortest edge 800
319
+ - **Hardware:** Hugging Face Jobs GPU (T4)
320
+ - **Selection:** best checkpoint by validation mAP@[0.50:0.95]
321
+
322
+ ## Evaluation Results (validation)
323
+
324
+ | Metric | Value |
325
+ |---|---|
326
+ | **mAP @[IoU=0.50:0.95]** | **{f('AP')}** |
327
+ | mAP @[IoU=0.50] | {f('AP50')} |
328
+ | mAP @[IoU=0.75] | {f('AP75')} |
329
+ | AP small | {f('AP_s')} |
330
+ | AP medium | {f('AP_m')} |
331
+ | AP large | {f('AP_l')} |
332
+ | AR (maxDets=100) | {f('AR_max100')} |
333
+
334
+ ## Usage
335
+
336
+ ```python
337
+ import torch
338
+ from transformers import DetrImageProcessor, DetrForObjectDetection
339
+ from PIL import Image
340
+
341
+ model = DetrForObjectDetection.from_pretrained("harness-race/opencode-r3")
342
+ processor = DetrImageProcessor.from_pretrained("harness-race/opencode-r3")
343
+ image = Image.open("page.jpg").convert("RGB")
344
+ inputs = processor(images=image, return_tensors="pt")
345
+ with torch.no_grad():
346
+ outputs = model(**inputs)
347
+ results = processor.post_process_object_detection(
348
+ outputs, target_sizes=[image.size[::-1]], threshold=0.5)[0]
349
+ for score, label_id, box in zip(results["scores"], results["labels"], results["boxes"]):
350
+ if score > 0.5:
351
+ print(model.config.id2label[label_id.item()], round(score.item(),3), box.tolist())
352
+ ```
353
+
354
+ ## Licensing
355
+
356
+ Base model `{ARGS.base_model}` is **Apache-2.0** (permissive open license), so this
357
+ fine-tuned model may be freely shared and reused. The training dataset is public
358
+ domain (**CC0-1.0**).
359
+
360
+ ## Known Limitations
361
+
362
+ - Trained on a single era/language; pre-1875 layouts may underperform.
363
+ - Skewed class distribution (headlines/ads dominate) can depress rare-class AP.
364
+ """
365
+
366
+ readme = build_readme(run_outputs["best"])
367
+ with open("README.md","w") as f:
368
+ f.write(readme)
369
+
370
+ # push README + a results.json
371
+ from huggingface_hub import HfApi
372
+ api = HfApi()
373
+ api.upload_file(path_or_fileobj=json.dumps(run_outputs["best"], indent=2), path_in_repo="validation_results.json", repo_id=ARGS.hub_id, repo_type="model")
374
+ api.upload_file(path_or_fileobj=readme.encode(), path_in_repo="README.md", repo_id=ARGS.hub_id, repo_type="model")
375
+ # model card metadata (config fields like license) already in config.json via push_to_hub
376
+
377
+ log("=== FINAL VALIDATION RESULTS ===")
378
+ print(json.dumps(run_outputs["best"], indent=2))
379
+ log("done")