Scientific-Table-Reconstruction / tools /tabulus_submission.py
jdsouza's picture
Document Chandra submissions through the integrated Tabulus adapter
8df7646 verified
Raw History Blame Contribute Delete
8.07 kB
"""Bridge the public dataset and Tabulus adapters without changing Tabulus source."""
import argparse
from collections import defaultdict
import csv
import hashlib
import json
from pathlib import Path, PurePosixPath
DATASET="SciKnowOrg/Scientific-Table-Reconstruction"
def write_json(path,value):
path.parent.mkdir(parents=True,exist_ok=True)
path.write_text(json.dumps(value,ensure_ascii=False,indent=2)+"\n",encoding="utf-8")
def canonical_hash(value):
text=json.dumps(value,sort_keys=True,ensure_ascii=False,allow_nan=False,separators=(",",":"))
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def stage(args):
directory=Path(args.dataset)
if not directory.is_dir():
from huggingface_hub import snapshot_download
directory=Path(snapshot_download(args.dataset,repo_type="dataset",revision="v1.0.1",
allow_patterns=["manifest.json","data/*.parquet"]))
import pyarrow.parquet as pq
manifest=json.loads((directory/"manifest.json").read_text(encoding="utf-8"))
rows=[]
for file in sorted((directory/"data").glob("*.parquet")):rows.extend(pq.read_table(file).to_pylist())
rows.sort(key=lambda row:(row["paper_id"],row["table_id"]))
content=[{"domain":row["domain"],"paper_id":row["paper_id"],"table_id":row["table_id"],
"image_sha256":hashlib.sha256(row["image"]["bytes"]).hexdigest(),
"gold_sha256":canonical_hash(row["gold_rows"])} for row in rows]
if canonical_hash(content)!=manifest["dataset_fingerprint"]:raise ValueError("Dataset fingerprint mismatch")
if args.out.exists() and any(args.out.iterdir()):raise ValueError("--out must be empty or new")
grouped=defaultdict(list)
for row in rows:grouped[row["paper_id"]].append(row)
mapping=[]
for pid,tables in grouped.items():
records=[]
for index,row in enumerate(tables,1):
relative="images/"+row["table_id"]+".png"
path=args.out/pid/relative
path.parent.mkdir(parents=True,exist_ok=True)
data=row["image"]["bytes"]
path.write_bytes(data)
records.append({"table_id":index,"image":relative,"paper_id":pid,"benchmark_table_key":row["table_id"]})
mapping.append({"paper_id":pid,"table_id":row["table_id"],"tabulus_table_id":index,
"image":pid+"/"+relative,"image_sha256":hashlib.sha256(data).hexdigest()})
write_json(args.out/pid/"tables_index.json",{"tables":records})
write_json(args.out/"submission_inputs.json",{"dataset":DATASET,"dataset_version":manifest["version"],
"dataset_fingerprint":manifest["dataset_fingerprint"],"tables":mapping})
print(f"Staged {len(rows)} original images from {len(grouped)} papers; no gold rows were written.")
def collect(args):
mapping=json.loads((args.inputs/"submission_inputs.json").read_text(encoding="utf-8"))
expected=defaultdict(dict)
for table in mapping["tables"]:
data=(args.inputs/table["image"]).read_bytes()
if hashlib.sha256(data).hexdigest()!=table["image_sha256"]:raise ValueError("An inference input changed")
expected[table["paper_id"]][table["tabulus_table_id"]]=table["table_id"]
output=[];audit=[]
saved=getattr(args,"saved_dataset",None)
saved_papers={}
if saved:
with (saved/"metadata"/"tables.csv").open(encoding="utf-8-sig",newline="") as handle:
for row in csv.DictReader(handle):
parts=Path(row["gold_table_path"]).parts
saved_papers[row["paper_id"]]=saved.joinpath(*parts[:parts.index("step2_table_reconstruction")])
for pid,indices in sorted(expected.items()):
directory=(saved_papers[pid]/"tabulus_runs"/"step2"/"controlled"/args.run/args.adapter/"reconstruction"
if saved else args.outputs/pid/args.adapter)
manifest_path=directory/"batch_summary.json"
if not manifest_path.exists():raise ValueError(f"Missing run manifest for {pid}; do not count unattempted tables as model errors")
summary=json.loads(manifest_path.read_text(encoding="utf-8"))
if summary["adapter_name"]!=args.adapter:raise ValueError(f"Wrong adapter for {pid}")
items=summary["items"]
if saved:
observed={}
for item in items:
source=PurePosixPath(item["source_image"].replace("\\","/"))
key=source.parent.name if source.stem=="crop" else source.stem
if key in observed:raise ValueError(f"Duplicate source identity for {pid}/{key}")
observed[key]=item
if set(observed)!=set(indices.values()):raise ValueError(f"Saved source IDs differ from released inputs for {pid}")
else:
observed={item["table_id"]:item for item in items}
if len(observed)!=len(items) or set(observed)!=set(indices):raise ValueError(f"Run IDs differ from staged inputs for {pid}")
for number,key in indices.items():
item=observed[key] if saved else observed[number]
status=item["status"]
rows=[];note=None
if status not in ("ok","empty","error"):raise ValueError(f"Unexpected status: {status}")
if status=="ok" and item.get("prediction_csv"):
path=(directory/item["prediction_csv"]).resolve()
if not path.is_relative_to(directory.resolve()):raise ValueError("Prediction path escapes its run folder")
if not path.is_file():raise ValueError(f"Missing saved prediction CSV for {pid}/{key}")
with path.open(encoding="utf-8-sig",newline="") as handle:rows=list(csv.reader(handle))
if not any(cell.strip() for row in rows for cell in row):status="empty";rows=[]
elif status=="ok":
status="empty";note="Adapter completed but produced no table CSV"
output.append({"paper_id":pid,"table_id":key,"status":status,"rows":rows})
audit.append({"paper_id":pid,"table_id":key,"adapter_status":item["status"],
"submission_status":status,"note":note})
args.out.parent.mkdir(parents=True,exist_ok=True)
if args.out.exists():raise ValueError("Refusing to overwrite an existing predictions file")
output.sort(key=lambda row:(row["paper_id"],row["table_id"]))
args.out.write_text("".join(json.dumps(row,ensure_ascii=False)+"\n" for row in output),encoding="utf-8")
write_json(args.out.with_suffix(".audit.json"),{"dataset_fingerprint":mapping["dataset_fingerprint"],
"adapter":args.adapter,"tables":audit})
from collections import Counter
print(json.dumps({"predictions":len(output),"status":Counter(row["status"] for row in output)}))
def main():
parser=argparse.ArgumentParser(description=__doc__)
commands=parser.add_subparsers(dest="command",required=True)
prepare=commands.add_parser("stage",help="Write image-only canonical crop folders accepted by Tabulus")
prepare.add_argument("--dataset",default=DATASET,help="HF dataset ID or local release directory")
prepare.add_argument("--out",type=Path,required=True)
prepare.set_defaults(function=stage)
export=commands.add_parser("collect",help="Package the Tabulus batch outputs as leaderboard predictions")
export.add_argument("--inputs",type=Path,required=True)
source=export.add_mutually_exclusive_group(required=True)
source.add_argument("--outputs",type=Path,help="Output parent from a new Tabulus --crops-folder run")
source.add_argument("--saved-dataset",type=Path,help="Existing TabulusBench tree containing controlled saved runs")
export.add_argument("--run",default="run_01",help="Explicit saved controlled run to reuse with --saved-dataset")
export.add_argument("--adapter",default="chandra")
export.add_argument("--out",type=Path,required=True)
export.set_defaults(function=collect)
args=parser.parse_args()
args.function(args)
if __name__=="__main__":main()