lev / questions.py

Lev 9B: frozen weights (rounds 1-6)

933eaed verified 16 days ago

15.9 kB

"""The four citator questions Lev answers about one (citing decision, cited case) pair.
Every system in the benchmark gets these same instructions and option descriptions.
"""

TREATMENT = {
    "followed": "The court treats the cited case as binding or adopts its rule as the governing rule, and decides the point with it.",
    "applied": "The court applies the cited case's rule or test to the facts before it.",
    "considered": "The court discusses the cited case's reasoning or scope at some length without adopting, distinguishing or rejecting it.",
    "distinguished": "The court says the cited case does not govern because the facts or the law differ.",
    "questioned": "The court doubts or criticises the cited case's correctness or reasoning but does not decline to follow it.",
    "not_followed": "The court declines to follow the cited case.",
    "overruled": "The court, having the power to do so, holds that the cited case was wrongly decided.",
    "superseded_by_statute": "The court says legislation has displaced the rule in the cited case.",
    "referred_to": "The court cites the case in its own reasoning for a point, without more (a 'see' citation, a restated principle, a list of authorities).",
    "inside_quotation": "The case appears only inside a passage that the court quotes from another source.",
    "other_court_or_party": "The case appears only where the court reports the reasoning of another court or the argument of a party, without adopting it.",
    "same_matter": "The cited case is an earlier decision in the same proceeding (the decision under appeal or review, or an earlier ruling between the same parties in this litigation).",
    "not_named": "The excerpt never actually names or cites this case.",
}

# When a decision treats one case in several ways, the label kept is the first of these that applies.
TREATMENT_PRIORITY = [
    "overruled", "not_followed", "superseded_by_statute", "questioned", "distinguished", "same_matter",
    "followed", "applied", "considered", "referred_to", "other_court_or_party", "inside_quotation", "not_named",
]
NEGATIVE = ("overruled", "not_followed", "superseded_by_statute", "questioned")

HISTORY = {
    "not_same_matter": "The cited case is not an earlier decision in this same proceeding.",
    "affirmed": "It is the decision under appeal or review, and this decision upholds it.",
    "reversed": "It is the decision under appeal or review, and this decision reverses it.",
    "reversed_in_part": "It is the decision under appeal or review, and this decision reverses part of it.",
    "varied": "It is the decision under appeal or review, and this decision varies it.",
    "set_aside": "It is the decision under appeal or review, and this decision sets it aside.",
    "restored": "It is an earlier decision in this proceeding that this decision restores.",
    "remitted": "It is the decision under appeal or review, and this decision sends the matter back to be decided again.",
    "earlier_ruling": "It is an earlier ruling in this same proceeding that this decision does not decide on.",
}

REPORTED = {
    "none": "The court does not report that anyone else has overruled, reversed, displaced, declined or doubted the cited case.",
    "overruled": "The court reports that another court overruled the cited case.",
    "reversed": "The court reports that the cited case was reversed on appeal.",
    "superseded_by_statute": "The court reports that legislation displaced the cited case.",
    "not_followed": "The court reports that another court declined to follow the cited case.",
    "doubted": "The court reports that another court doubted the cited case.",
}

QUESTIONS = {
    "treatment": {
        "type": "choice",
        "instructions": "How does the citing decision treat the cited case? If it treats the case in more than one way, choose the most negative treatment.",
        "criteria": TREATMENT,
    },
    "history": {
        "type": "choice",
        "instructions": "Is the cited case an earlier decision in the same proceeding as the citing decision, and if so, what does the citing decision do to it?",
        "criteria": HISTORY,
    },
    "reported": {
        "type": "choice",
        "instructions": "Does the citing decision report that another court or legislation (not the citing court itself) has overruled, reversed, displaced, declined to follow or doubted the cited case?",
        "criteria": REPORTED,
    },
    "dissent": {
        "type": "noul",
        "instructions": "Do dissenting reasons in the citing decision cite or treat the cited case?",
    },
}


# ---- Decision-level questions (state: header, front matter, opening and closing paragraphs) ----

OUTCOME = {
    "allowed": "An appeal or review is allowed.",
    "allowed_in_part": "An appeal or review is allowed in part.",
    "dismissed": "The appeal, application, motion or claim is dismissed.",
    "granted": "A motion, application or claim is granted.",
    "granted_in_part": "A motion, application or claim is granted in part.",
    "refused": "Leave, bail or another discretionary request is refused.",
    "quashed": "A decision, conviction or order is quashed.",
    "new_trial": "A new trial or new hearing is ordered.",
    "remitted": "The matter is sent back to the decision-maker below.",
    "stayed": "The proceeding or the charge is stayed.",
    "varied": "An order or sentence is varied.",
    "convicted": "The accused is found guilty.",
    "acquitted": "The accused is found not guilty.",
    "sentenced": "A sentence is imposed.",
    "other": "Some other disposition.",
    "none": "The decision makes no disposition.",
}

DECISION_TYPE = {
    "reasons": "Full written reasons for decision.",
    "endorsement": "A short endorsement or brief disposition.",
    "oral_reasons": "Reasons delivered orally.",
    "ruling": "A ruling made during a hearing, for example on a voir dire or an objection.",
    "costs": "Reasons on costs only.",
    "sentencing": "Reasons on sentence only.",
    "leave": "A decision on leave to appeal.",
    "supplementary": "Supplementary or additional reasons.",
    "corrigendum": "A correction to earlier reasons.",
    "other": "None of the above.",
}

PROCEEDING = {
    "appeal": "An appeal to this court.",
    "judicial_review": "Judicial review of an administrative decision.",
    "trial": "A trial or a hearing on the merits in a court.",
    "motion": "A motion.",
    "application": "An application.",
    "sentencing": "A sentencing hearing.",
    "bail": "A bail hearing or bail review.",
    "leave": "An application for leave to appeal.",
    "reference": "A reference.",
    "hearing": "A hearing before a tribunal.",
    "other": "None of the above.",
}

AREAS = [
    "Aboriginal and Indigenous Law",
    "Access to Information and Privacy",
    "Administrative Law",
    "Arbitration and Dispute Resolution",
    "Bankruptcy and Insolvency",
    "Citizenship, Immigration and Refugee Law",
    "Civil Procedure",
    "Commercial Law",
    "Competition, Trade and Customs",
    "Conflict of Laws",
    "Constitutional Law",
    "Construction Law",
    "Contracts",
    "Corporations and Business Associations",
    "Criminal Law",
    "Criminal Procedure",
    "Damages and Remedies",
    "Education Law",
    "Elections and Democratic Institutions",
    "Employment Law",
    "Labour Relations",
    "Workers' Compensation and Workplace Safety",
    "Environmental Law",
    "Estates and Trusts",
    "Evidence",
    "Family Law",
    "Health Law",
    "Mental Health, Capacity and Guardianship",
    "Human Rights",
    "Insurance",
    "Intellectual Property",
    "International Law",
    "Maritime Law",
    "Military Law",
    "Motor Vehicles and Highway Traffic",
    "Municipal and Planning Law",
    "Natural Resources, Energy and Agriculture",
    "Professions and Occupations",
    "Real Property",
    "Residential Tenancies",
    "Restitution and Equity",
    "Securities and Financial Services",
    "Sentencing",
    "Social Benefits",
    "Statutory Interpretation",
    "Taxation",
    "Torts",
    "Transportation and Communications",
    "Youth Criminal Justice"
]

STANDARD_OF_REVIEW = {
    "correctness": "The court applies the correctness standard to this issue.",
    "reasonableness": "The court applies the reasonableness standard to this issue.",
    "palpable_and_overriding_error": "The court applies the palpable and overriding error standard to this issue.",
    "patent_unreasonableness": "The court applies the patent unreasonableness standard to this issue.",
    "other": "The court states some other standard of review for this issue.",
    "not_stated": "The excerpt states no standard of review for this issue.",
}

ELEMENT = {
    "met": "The court finds that the condition this element states is satisfied on the facts.",
    "not_met": "The court finds that the condition this element states is not satisfied.",
    "not_decided": "The court does not decide this element.",
}

QUESTIONS.update({
    "outcome": {"type": "choice", "instructions": "What is the disposition in this decision?", "criteria": OUTCOME},
    "decision_type": {"type": "choice", "instructions": "What kind of document is this decision?", "criteria": DECISION_TYPE},
    "proceeding": {"type": "choice", "instructions": "What kind of proceeding does this decision decide?", "criteria": PROCEEDING},
    "area": {"type": "choice", "instructions": "What is the area of law of the main issue this decision decides? For an appeal, choose the area of the dispute under appeal.",
             "criteria": {a: None for a in AREAS}},
    "publication_ban": {"type": "noul", "instructions": "Does the decision state that a publication ban, a sealing order or statutory anonymisation applies?"},
    "self_represented": {"type": "noul", "instructions": "Does the decision say that a party was self-represented or appeared in person?"},
    "separate_opinions": {"type": "noul", "instructions": "Does the decision include separate concurring or dissenting reasons?"},
    "standard_of_review": {"type": "choice", "instructions": "Which standard of review does the court apply to the issue stated in the excerpt?", "criteria": STANDARD_OF_REVIEW},
    "element_finding": {"type": "choice", "instructions": "What does the court find on the element of the legal test stated in the excerpt?", "criteria": ELEMENT},
})

CITATOR = ("treatment", "history", "reported", "dissent")
DECISION_LEVEL = ("outcome", "decision_type", "proceeding", "area", "publication_ban", "self_represented", "separate_opinions")
PASSAGE_LEVEL = ("standard_of_review", "element_finding")


# ---- Second-generation questions ----

VOICE = {
    "own_reasoning": "The court cites or discusses the case in its own reasoning.",
    "inside_quotation": "The case appears only inside a passage the court quotes from another source.",
    "other_court_or_party": "The case appears only where the court reports another court's reasoning or a party's argument.",
    "not_named": "The excerpt never actually names or cites this case.",
}
TREATMENT_GROUP = {
    "positive": "The court follows the case or applies its rule.",
    "neutral": "The court considers the case or merely refers to it.",
    "distinguished": "The court says the case does not govern because the facts or the law differ.",
    "negative": "The court questions the case, declines to follow it, overrules it, or holds it superseded by statute.",
    "not_courts_own": "The case appears only in a quotation, in a party's argument or another court's reasoning, or not at all.",
    "same_matter": "The case is an earlier decision in the same proceeding.",
}
GROUP_OF = {"followed": "positive", "applied": "positive", "considered": "neutral", "referred_to": "neutral",
            "distinguished": "distinguished", "questioned": "negative", "not_followed": "negative", "overruled": "negative",
            "superseded_by_statute": "negative", "inside_quotation": "not_courts_own", "other_court_or_party": "not_courts_own",
            "not_named": "not_courts_own", "same_matter": "same_matter"}
VOICE_OF = {**{k: "own_reasoning" for k in GROUP_OF}, "inside_quotation": "inside_quotation",
            "other_court_or_party": "other_court_or_party", "not_named": "not_named"}
BURDEN = {
    "balance_of_probabilities": "The court applies the balance of probabilities to this issue.",
    "beyond_reasonable_doubt": "The court applies proof beyond a reasonable doubt to this issue.",
    "other": "The court states some other burden or standard of proof for this issue.",
    "not_stated": "The excerpt states no burden or standard of proof for this issue.",
}
PROVISION = {
    "interpreted": "The court states what the provision's words mean, how far it reaches, or what its purpose requires.",
    "applied": "The court applies the provision to the facts without stating its meaning.",
    "mentioned": "The court only refers to the provision or quotes it.",
}
PREVAILING = {
    "appellant": None, "respondent": None, "applicant": None, "plaintiff": None, "defendant": None, "accused": None,
    "crown": None, "split": "The result is mixed.", "other": "Another party, or unclear.",
}
COUNSEL = {
    "listed": "Counsel are named.",
    "unnamed_representative": "A party was represented but the representative is not named.",
    "none": "The decision says a party was unrepresented and no counsel is named.",
    "redacted": "Names of counsel were removed or reduced to initials.",
    "not_stated": "There is no appearance information at all.",
}

QUESTIONS.update({
    "voice": {"type": "choice", "instructions": "In whose voice does the cited case appear in the citing decision?", "criteria": VOICE},
    "negative": {"type": "noul", "instructions": "Does the citing court itself treat the cited case negatively: question it, decline to follow it, overrule it, or hold it superseded by statute?"},
    "treatment_group": {"type": "choice", "instructions": "Broadly, how does the citing decision treat the cited case? If it treats the case in more than one way, choose the most negative.", "criteria": TREATMENT_GROUP},
    "burden": {"type": "choice", "instructions": "Which burden or standard of proof does the court apply to the issue stated in the excerpt?", "criteria": BURDEN},
    "provision_treatment": {"type": "choice", "instructions": "How does the decision treat the statutory provision named in the excerpt?", "criteria": PROVISION},
    "proposition": {"type": "noul", "instructions": "Does the court, in this paragraph, state a proposition of law that it relies on to decide?"},
    "prevailing_party": {"type": "choice", "instructions": "Which party prevailed in this decision?", "criteria": PREVAILING},
    "counsel_status": {"type": "choice", "instructions": "What does the decision say about who appeared for the parties?", "criteria": COUNSEL},
})
AUXILIARY = ("voice", "negative", "treatment_group")
SECOND_GENERATION = ("burden", "provision_treatment", "proposition", "prevailing_party", "counsel_status", "cited_for", "cite_fill")

# questions whose options come with the example
ITEM_INSTRUCTIONS = {
    "cite_fill": "One citation was removed from the paragraph and replaced with [CITATION]. Which case was cited there?",
    "cited_for": "What does the citing decision cite the case for?",
    "au_treatment": "How does the citing decision treat the cited case? If it treats the case in more than one way, choose the most negative treatment.",
}


def spec_of(row):
    """The question of a training or test row: a fixed one from QUESTIONS, or one the row carries itself."""
    if row.get("options"):
        return {"type": "choice", "instructions": row.get("instructions") or ITEM_INSTRUCTIONS[row["q"]], "criteria": dict(row["options"])}
    if row.get("instructions"):
        return {"type": "noul", "instructions": row["instructions"]}
    return QUESTIONS[row["q"]]

	"""The four citator questions Lev answers about one (citing decision, cited case) pair.

	Every system in the benchmark gets these same instructions and option descriptions.
	"""

	TREATMENT = {
	"followed": "The court treats the cited case as binding or adopts its rule as the governing rule, and decides the point with it.",
	"applied": "The court applies the cited case's rule or test to the facts before it.",
	"considered": "The court discusses the cited case's reasoning or scope at some length without adopting, distinguishing or rejecting it.",
	"distinguished": "The court says the cited case does not govern because the facts or the law differ.",
	"questioned": "The court doubts or criticises the cited case's correctness or reasoning but does not decline to follow it.",
	"not_followed": "The court declines to follow the cited case.",
	"overruled": "The court, having the power to do so, holds that the cited case was wrongly decided.",
	"superseded_by_statute": "The court says legislation has displaced the rule in the cited case.",
	"referred_to": "The court cites the case in its own reasoning for a point, without more (a 'see' citation, a restated principle, a list of authorities).",
	"inside_quotation": "The case appears only inside a passage that the court quotes from another source.",
	"other_court_or_party": "The case appears only where the court reports the reasoning of another court or the argument of a party, without adopting it.",
	"same_matter": "The cited case is an earlier decision in the same proceeding (the decision under appeal or review, or an earlier ruling between the same parties in this litigation).",
	"not_named": "The excerpt never actually names or cites this case.",
	}

	# When a decision treats one case in several ways, the label kept is the first of these that applies.
	TREATMENT_PRIORITY = [
	"overruled", "not_followed", "superseded_by_statute", "questioned", "distinguished", "same_matter",
	"followed", "applied", "considered", "referred_to", "other_court_or_party", "inside_quotation", "not_named",
	]
	NEGATIVE = ("overruled", "not_followed", "superseded_by_statute", "questioned")

	HISTORY = {
	"not_same_matter": "The cited case is not an earlier decision in this same proceeding.",
	"affirmed": "It is the decision under appeal or review, and this decision upholds it.",
	"reversed": "It is the decision under appeal or review, and this decision reverses it.",
	"reversed_in_part": "It is the decision under appeal or review, and this decision reverses part of it.",
	"varied": "It is the decision under appeal or review, and this decision varies it.",
	"set_aside": "It is the decision under appeal or review, and this decision sets it aside.",
	"restored": "It is an earlier decision in this proceeding that this decision restores.",
	"remitted": "It is the decision under appeal or review, and this decision sends the matter back to be decided again.",
	"earlier_ruling": "It is an earlier ruling in this same proceeding that this decision does not decide on.",
	}

	REPORTED = {
	"none": "The court does not report that anyone else has overruled, reversed, displaced, declined or doubted the cited case.",
	"overruled": "The court reports that another court overruled the cited case.",
	"reversed": "The court reports that the cited case was reversed on appeal.",
	"superseded_by_statute": "The court reports that legislation displaced the cited case.",
	"not_followed": "The court reports that another court declined to follow the cited case.",
	"doubted": "The court reports that another court doubted the cited case.",
	}

	QUESTIONS = {
	"treatment": {
	"type": "choice",
	"instructions": "How does the citing decision treat the cited case? If it treats the case in more than one way, choose the most negative treatment.",
	"criteria": TREATMENT,
	},
	"history": {
	"type": "choice",
	"instructions": "Is the cited case an earlier decision in the same proceeding as the citing decision, and if so, what does the citing decision do to it?",
	"criteria": HISTORY,
	},
	"reported": {
	"type": "choice",
	"instructions": "Does the citing decision report that another court or legislation (not the citing court itself) has overruled, reversed, displaced, declined to follow or doubted the cited case?",
	"criteria": REPORTED,
	},
	"dissent": {
	"type": "noul",
	"instructions": "Do dissenting reasons in the citing decision cite or treat the cited case?",
	},
	}


	# ---- Decision-level questions (state: header, front matter, opening and closing paragraphs) ----

	OUTCOME = {
	"allowed": "An appeal or review is allowed.",
	"allowed_in_part": "An appeal or review is allowed in part.",
	"dismissed": "The appeal, application, motion or claim is dismissed.",
	"granted": "A motion, application or claim is granted.",
	"granted_in_part": "A motion, application or claim is granted in part.",
	"refused": "Leave, bail or another discretionary request is refused.",
	"quashed": "A decision, conviction or order is quashed.",
	"new_trial": "A new trial or new hearing is ordered.",
	"remitted": "The matter is sent back to the decision-maker below.",
	"stayed": "The proceeding or the charge is stayed.",
	"varied": "An order or sentence is varied.",
	"convicted": "The accused is found guilty.",
	"acquitted": "The accused is found not guilty.",
	"sentenced": "A sentence is imposed.",
	"other": "Some other disposition.",
	"none": "The decision makes no disposition.",
	}

	DECISION_TYPE = {
	"reasons": "Full written reasons for decision.",
	"endorsement": "A short endorsement or brief disposition.",
	"oral_reasons": "Reasons delivered orally.",
	"ruling": "A ruling made during a hearing, for example on a voir dire or an objection.",
	"costs": "Reasons on costs only.",
	"sentencing": "Reasons on sentence only.",
	"leave": "A decision on leave to appeal.",
	"supplementary": "Supplementary or additional reasons.",
	"corrigendum": "A correction to earlier reasons.",
	"other": "None of the above.",
	}

	PROCEEDING = {
	"appeal": "An appeal to this court.",
	"judicial_review": "Judicial review of an administrative decision.",
	"trial": "A trial or a hearing on the merits in a court.",
	"motion": "A motion.",
	"application": "An application.",
	"sentencing": "A sentencing hearing.",
	"bail": "A bail hearing or bail review.",
	"leave": "An application for leave to appeal.",
	"reference": "A reference.",
	"hearing": "A hearing before a tribunal.",
	"other": "None of the above.",
	}

	AREAS = [
	"Aboriginal and Indigenous Law",
	"Access to Information and Privacy",
	"Administrative Law",
	"Arbitration and Dispute Resolution",
	"Bankruptcy and Insolvency",
	"Citizenship, Immigration and Refugee Law",
	"Civil Procedure",
	"Commercial Law",
	"Competition, Trade and Customs",
	"Conflict of Laws",
	"Constitutional Law",
	"Construction Law",
	"Contracts",
	"Corporations and Business Associations",
	"Criminal Law",
	"Criminal Procedure",
	"Damages and Remedies",
	"Education Law",
	"Elections and Democratic Institutions",
	"Employment Law",
	"Labour Relations",
	"Workers' Compensation and Workplace Safety",
	"Environmental Law",
	"Estates and Trusts",
	"Evidence",
	"Family Law",
	"Health Law",
	"Mental Health, Capacity and Guardianship",
	"Human Rights",
	"Insurance",
	"Intellectual Property",
	"International Law",
	"Maritime Law",
	"Military Law",
	"Motor Vehicles and Highway Traffic",
	"Municipal and Planning Law",
	"Natural Resources, Energy and Agriculture",
	"Professions and Occupations",
	"Real Property",
	"Residential Tenancies",
	"Restitution and Equity",
	"Securities and Financial Services",
	"Sentencing",
	"Social Benefits",
	"Statutory Interpretation",
	"Taxation",
	"Torts",
	"Transportation and Communications",
	"Youth Criminal Justice"
	]

	STANDARD_OF_REVIEW = {
	"correctness": "The court applies the correctness standard to this issue.",
	"reasonableness": "The court applies the reasonableness standard to this issue.",
	"palpable_and_overriding_error": "The court applies the palpable and overriding error standard to this issue.",
	"patent_unreasonableness": "The court applies the patent unreasonableness standard to this issue.",
	"other": "The court states some other standard of review for this issue.",
	"not_stated": "The excerpt states no standard of review for this issue.",
	}

	ELEMENT = {
	"met": "The court finds that the condition this element states is satisfied on the facts.",
	"not_met": "The court finds that the condition this element states is not satisfied.",
	"not_decided": "The court does not decide this element.",
	}

	QUESTIONS.update({
	"outcome": {"type": "choice", "instructions": "What is the disposition in this decision?", "criteria": OUTCOME},
	"decision_type": {"type": "choice", "instructions": "What kind of document is this decision?", "criteria": DECISION_TYPE},
	"proceeding": {"type": "choice", "instructions": "What kind of proceeding does this decision decide?", "criteria": PROCEEDING},
	"area": {"type": "choice", "instructions": "What is the area of law of the main issue this decision decides? For an appeal, choose the area of the dispute under appeal.",
	"criteria": {a: None for a in AREAS}},
	"publication_ban": {"type": "noul", "instructions": "Does the decision state that a publication ban, a sealing order or statutory anonymisation applies?"},
	"self_represented": {"type": "noul", "instructions": "Does the decision say that a party was self-represented or appeared in person?"},
	"separate_opinions": {"type": "noul", "instructions": "Does the decision include separate concurring or dissenting reasons?"},
	"standard_of_review": {"type": "choice", "instructions": "Which standard of review does the court apply to the issue stated in the excerpt?", "criteria": STANDARD_OF_REVIEW},
	"element_finding": {"type": "choice", "instructions": "What does the court find on the element of the legal test stated in the excerpt?", "criteria": ELEMENT},
	})

	CITATOR = ("treatment", "history", "reported", "dissent")
	DECISION_LEVEL = ("outcome", "decision_type", "proceeding", "area", "publication_ban", "self_represented", "separate_opinions")
	PASSAGE_LEVEL = ("standard_of_review", "element_finding")


	# ---- Second-generation questions ----

	VOICE = {
	"own_reasoning": "The court cites or discusses the case in its own reasoning.",
	"inside_quotation": "The case appears only inside a passage the court quotes from another source.",
	"other_court_or_party": "The case appears only where the court reports another court's reasoning or a party's argument.",
	"not_named": "The excerpt never actually names or cites this case.",
	}
	TREATMENT_GROUP = {
	"positive": "The court follows the case or applies its rule.",
	"neutral": "The court considers the case or merely refers to it.",
	"distinguished": "The court says the case does not govern because the facts or the law differ.",
	"negative": "The court questions the case, declines to follow it, overrules it, or holds it superseded by statute.",
	"not_courts_own": "The case appears only in a quotation, in a party's argument or another court's reasoning, or not at all.",
	"same_matter": "The case is an earlier decision in the same proceeding.",
	}
	GROUP_OF = {"followed": "positive", "applied": "positive", "considered": "neutral", "referred_to": "neutral",
	"distinguished": "distinguished", "questioned": "negative", "not_followed": "negative", "overruled": "negative",
	"superseded_by_statute": "negative", "inside_quotation": "not_courts_own", "other_court_or_party": "not_courts_own",
	"not_named": "not_courts_own", "same_matter": "same_matter"}
	VOICE_OF = {**{k: "own_reasoning" for k in GROUP_OF}, "inside_quotation": "inside_quotation",
	"other_court_or_party": "other_court_or_party", "not_named": "not_named"}
	BURDEN = {
	"balance_of_probabilities": "The court applies the balance of probabilities to this issue.",
	"beyond_reasonable_doubt": "The court applies proof beyond a reasonable doubt to this issue.",
	"other": "The court states some other burden or standard of proof for this issue.",
	"not_stated": "The excerpt states no burden or standard of proof for this issue.",
	}
	PROVISION = {
	"interpreted": "The court states what the provision's words mean, how far it reaches, or what its purpose requires.",
	"applied": "The court applies the provision to the facts without stating its meaning.",
	"mentioned": "The court only refers to the provision or quotes it.",
	}
	PREVAILING = {
	"appellant": None, "respondent": None, "applicant": None, "plaintiff": None, "defendant": None, "accused": None,
	"crown": None, "split": "The result is mixed.", "other": "Another party, or unclear.",
	}
	COUNSEL = {
	"listed": "Counsel are named.",
	"unnamed_representative": "A party was represented but the representative is not named.",
	"none": "The decision says a party was unrepresented and no counsel is named.",
	"redacted": "Names of counsel were removed or reduced to initials.",
	"not_stated": "There is no appearance information at all.",
	}

	QUESTIONS.update({
	"voice": {"type": "choice", "instructions": "In whose voice does the cited case appear in the citing decision?", "criteria": VOICE},
	"negative": {"type": "noul", "instructions": "Does the citing court itself treat the cited case negatively: question it, decline to follow it, overrule it, or hold it superseded by statute?"},
	"treatment_group": {"type": "choice", "instructions": "Broadly, how does the citing decision treat the cited case? If it treats the case in more than one way, choose the most negative.", "criteria": TREATMENT_GROUP},
	"burden": {"type": "choice", "instructions": "Which burden or standard of proof does the court apply to the issue stated in the excerpt?", "criteria": BURDEN},
	"provision_treatment": {"type": "choice", "instructions": "How does the decision treat the statutory provision named in the excerpt?", "criteria": PROVISION},
	"proposition": {"type": "noul", "instructions": "Does the court, in this paragraph, state a proposition of law that it relies on to decide?"},
	"prevailing_party": {"type": "choice", "instructions": "Which party prevailed in this decision?", "criteria": PREVAILING},
	"counsel_status": {"type": "choice", "instructions": "What does the decision say about who appeared for the parties?", "criteria": COUNSEL},
	})
	AUXILIARY = ("voice", "negative", "treatment_group")
	SECOND_GENERATION = ("burden", "provision_treatment", "proposition", "prevailing_party", "counsel_status", "cited_for", "cite_fill")

	# questions whose options come with the example
	ITEM_INSTRUCTIONS = {
	"cite_fill": "One citation was removed from the paragraph and replaced with [CITATION]. Which case was cited there?",
	"cited_for": "What does the citing decision cite the case for?",
	"au_treatment": "How does the citing decision treat the cited case? If it treats the case in more than one way, choose the most negative treatment.",
	}


	def spec_of(row):
	"""The question of a training or test row: a fixed one from QUESTIONS, or one the row carries itself."""
	if row.get("options"):
	return {"type": "choice", "instructions": row.get("instructions") or ITEM_INSTRUCTIONS[row["q"]], "criteria": dict(row["options"])}
	if row.get("instructions"):
	return {"type": "noul", "instructions": row["instructions"]}
	return QUESTIONS[row["q"]]