VibeHash's picture
Release laya-ledgar, run ce-seed0
779ff0b verified
Raw History Blame Contribute Delete
3.47 kB
"""LEDGAR: which of 100 kinds of contract provision a paragraph from a real contract is.
The LexGLUE version (Chalkidis et al. 2022): 60,000 training and 10,000 test provisions from contracts filed with
the US Securities and Exchange Commission, each labelled with its most frequent heading type. Every provision has
exactly one label, so micro-F1, the number LexGLUE reports first, equals accuracy. LexGLUE also reports macro-F1,
the F1 of each provision type averaged with equal weight, because the types are very unequal in size. This task
reports both.
Option texts are the label names in lower case.
Contracts repeat boilerplate, so many test provisions also appear word for word in training. They are left in, as in
the official split, and the scorer reports accuracy on unseen provisions apart.
"""
from __future__ import annotations
from typing import Any, Dict, List, Optional
from layatasks import metrics
from layatasks.task import Task
def option_text(name: str) -> str:
return name.strip().lower()
TASK = Task(
name="ledgar",
title="LEDGAR",
repo="coastalcph/lex_glue",
revision="c23fdff1a6bf74e0e1a71cb86f1e781d37da888c",
license="CC BY 4.0",
attribution=("Tuggener et al., LEDGAR: A Large-Scale Multi-label Corpus for Text Classification of Legal "
"Provisions in Contracts, LREC 2020; Chalkidis et al., LexGLUE: A Benchmark Dataset for Legal "
"Language Understanding in English, ACL 2022. Dataset: coastalcph/lex_glue (ledgar) on Hugging "
"Face, CC BY 4.0 per its dataset card."),
files={
"train": ("ledgar/train-00000-of-00001.parquet",
"e2901d9e019f5652dadda839d488155c681e50b90bd33f00574949418827867d"),
"test": ("ledgar/test-00000-of-00001.parquet",
"af9ed0ec09545d39449ffddca3996a1a7774e2bd0a9d6fc09d4e9dd067892a7a"),
},
text_column="text",
label_column="label",
question_id="provision",
instructions="Which kind of contract provision is this?",
option_text=option_text,
test_sha256="8c3aa107260838655efebe9f319a60258b0a6e2ee3ec156314bd91b544c5a2e0",
# The 100 options take 286 tokens. At Laya's default max_len of 512 that leaves 225 for the provision and cuts
# 16.4% of test provisions short; at 768 it leaves 481, about the text budget LexGLUE's 512-token models had,
# and cuts 2.5%. Measured by layatasks.fit before any model was trained.
max_len=768,
# Changed after the first run, which used Laya's rlcd loss for 2 epochs and was scored once on the test: 79.9%.
# That run got only 81.8% of 2,000 of its own training rows right, so it had not fitted them. Short probes then
# trained on 8,000 training rows and scored 2,000 others, the test unread: rlcd 73.2%, plain cross-entropy
# ("soft-ce") 78.3%, soft-ce at twice the learning rates 78.4%. So soft-ce at Laya's own rates and Laya's own 4
# epochs. 2,000 training rows are kept out to judge the run before the test is read a second time.
epochs=4,
loss="soft-ce",
dev_rows=2000,
release_name="laya-ledgar",
)
def extra_scores(preds: List[Dict[str, Any]], gate: Optional[float]) -> Dict[str, Any]:
"""LexGLUE's two numbers: micro-F1 (equal to accuracy here) and macro-F1."""
golds = [p["gold"] for p in preds]
picks = [p["pred"] for p in preds]
return {"micro_f1": metrics.micro_f1(golds, picks), "macro_f1": metrics.macro_f1(golds, picks)}