docs/spec/sources.json

{
  "DS-A": {
    "name": "CT Open Expenditures — Ledger (Current)",
    "org": "State of Connecticut",
    "url": "https://data.ct.gov/api/views/ajdm-rvz7",
    "access": "CSV / JSON / XML + Socrata API. Updated nightly. Dataset id ajdm-rvz7.",
    "fields": "fiscal_year, fiscal_year_period, service, department, program, account, expense_category, fund_code, fund, fund_type, vendor, vendor_id, payment_id, payment_count, payment_method, payment_status, payment_date, invoice_id, invoice_line_number, invoice_line_distribution_number, invoice_date, amount, description, contract_number, purchase_order_number, purchase_order_line_number, vendor_type, payment_description, load_dttm",
    "why": "The richest public source for this work. Has invoice date AND payment date AND fiscal period (cut-off tasks), PO and contract numbers (three-way match), vendor id and type (vendor anomalies), account/fund/department strings (coding validity), and a load timestamp (posting-behaviour anomalies).",
    "role": "PRIMARY BASE LEDGER",
    "ingest": {
      "id": "DS-A",
      "status": "VERIFIED LIVE 2026-08-03",
      "vol": "6,128,020 rows · FY2020–FY2027 · ~860k–900k rows and $29–40B per fiscal year",
      "how": "Socrata REST. Base: https://data.ct.gov/resource/ajdm-rvz7.json — supports $select, $where, $group, $order, $limit, $offset. Bulk CSV also available from the dataset page.",
      "snippet": "# count\ncurl 'https://data.ct.gov/resource/ajdm-rvz7.json?$select=count(*)'\n# one fiscal period, paged\ncurl 'https://data.ct.gov/resource/ajdm-rvz7.json?$where=fiscal_year=2026 AND fiscal_year_period=7&$limit=50000&$offset=0'\n# group-by works; count(distinct x) and $query do NOT on this endpoint",
      "gotchas": [
        "EVERY value returns as a JSON string, including amount and all dates — cast on ingest or every numeric comparison silently fails.",
        "Two datetime formats in one row: payment_date / invoice_date are ISO with .000 suffix; load_dttm is 'YYYY-MM-DD HH:MM:SS.ffffff' with a space. Parse separately.",
        "contract_number is sparse — present on some rows, absent entirely on others. Filter to non-null for JE-06 contract variants.",
        "vendor_type is null on 322,154 rows (~5%). Real missing-data noise: keep it, do not impute — it is a free distractor.",
        "payment_status has 124,780 rows valued '-' plus 3,019 Void-Reconciled and 5,223 Stopped-*. The voids are the natural void/reissue population for JE-04.",
        "FY2027 is partial (52,605 rows at time of check) — exclude or the volume profile skews.",
        "Updated nightly, so row counts drift. Snapshot to Parquet once and version it, or task instances are not reproducible."
      ],
      "licence": "State of Connecticut open data. No stated restriction; attribute the source."
    }
  },
  "DS-B": {
    "name": "City of Houston Checkbook",
    "org": "City of Houston Enterprise Data Office",
    "url": "https://data.houstontx.gov/dataset/checkbook",
    "access": "CSV, one file per year, 2018–present. ODC-BY licence.",
    "fields": "18 columns, Title Case with spaces: Payment Document Number, Fund ID/Name, Department ID/Name, WBS ID/Description, GL Account Number/Description, Vendor Name, Vendor Invoice, Fiscal Year, Clearing Date, Amount, Type of procurement, Purchase Order Number/Item, Contract Number. Pinned verbatim in app/src/je_validation/ingest/ds_b/schema.py.",
    "why": "Second independent real ledger. Use it as a held-out distribution so tasks tuned on CT do not overfit one entity's coding conventions.",
    "role": "HELD-OUT BASE LEDGER",
    "ingest": {
      "id": "DS-B",
      "status": "VERIFIED LIVE 2026-08-05 · snapshot frozen (manifest ds_b_2026-08-05)",
      "vol": "2,293,699 rows · FY2018–2026 · 9 per-year CSVs",
      "how": "CKAN dataset page: https://data.houstontx.gov/dataset/checkbook — download per-year CSVs.",
      "snippet": "# headers pinned in app/src/je_validation/ingest/ds_b/schema.py\nhead -1 checkbook-2026.csv",
      "gotchas": [
        "18 Title Case columns; the rename layer in app/src/je_validation/ingest/translate.py maps 14 of DS-A's 29 typed columns — payment_status, vendor_id, invoice_date and 12 others have no counterpart.",
        "Amounts may carry thousands separators; the cast strips them under an exact-cents guard.",
        "A Power BI report is published alongside; ignore it, take the CSVs."
      ],
      "licence": "Open Data Commons Attribution Licence (ODC-BY) — attribution required on any derived data you distribute."
    }
  },
  "DS-C": {
    "name": "Open Checkbook BR (Baton Rouge)",
    "org": "City of Baton Rouge / East Baton Rouge Parish, Finance",
    "url": "https://data.brla.gov/resource/7qhq-wwsg",
    "access": "Socrata CSV / JSON export. 787,690 rows, 2018–2026, 29 columns. Verified live 2026-08-05.",
    "fields": "jrnl_entry_year/per, fund + seg ids, department/function/division, invoice_status, check_number/date, invoice_number/date, line_number, line_item_amount/desc, source_document + type/desc, object (expense category id) + desc, commodity, vendor_name/zip, payment_type, payment_status.",
    "why": "Held-out second real ledger, chosen for carrying what Houston and Boston lack: payment status (zero blanks), invoice date/number/line (near-zero blanks), a flagged void population (2,509 'V' invoices), accounting periods, and a populated description field. 15 of DS-A's 29 typed columns map by rename; its year/period fields are calendar-basis and deliberately stay source-named.",
    "role": "HELD-OUT BASE LEDGER",
    "ingest": {
      "id": "DS-C",
      "status": "VERIFIED LIVE 2026-08-05",
      "vol": "787,690 rows · 2018–2026 · 2026 partial (41,856 rows)",
      "how": "Socrata REST. Base: https://data.brla.gov/resource/7qhq-wwsg.json — same query surface as DS-A. CSV export: https://data.brla.gov/resource/7qhq-wwsg.csv",
      "snippet": "# count\ncurl 'https://data.brla.gov/resource/7qhq-wwsg.json?$select=count(*)'\n# one journal-entry year\ncurl \"https://data.brla.gov/resource/7qhq-wwsg.csv?\\$where=jrnl_entry_year=2024&\\$order=:id&\\$limit=1000000\"",
      "gotchas": [
        "Every value returns as a string — cast on ingest, exactly like DS-A.",
        "check_date / invoice_date are ISO midnight timestamps with .000 — DS-A's date cast and midnight guard port directly.",
        "object and seg1 are typed 'number' in portal metadata but are account/fund identifiers — keep them String.",
        "Years are journal-entry (calendar) years, NOT July–June fiscal years like DS-A and Houston — never compare periods across datasets without a calendar mapping.",
        "2026 is partial — the detect-partial logic must flag it.",
        "This portal's SoQL rejects round(); do decimal checks at ingest, not in the API."
      ],
      "licence": "Public Domain (portal metadata; attribution field 'Finance')."
    }
  },
  "DS-D": {
    "name": "Delphi General Ledger (USDOT)",
    "org": "US DOT / Office of the Secretary, via Virginia Open Data",
    "url": "https://data.virginia.gov/dataset/delphi-general-ledger",
    "access": "HTML resource redirecting to the USDOT data inventory; request the extract.",
    "fields": "USSGL chart of accounts with actual, budget and encumbrance balances, event timestamps, journal entry source information, ledger relationships.",
    "why": "The only listed public source that carries budget and encumbrance balances alongside actuals plus JE source info — the substrate for obligation/encumbrance liquidation tasks.",
    "role": "BUDGET + ENCUMBRANCE LAYER",
    "ingest": {
      "id": "DS-D",
      "status": "NOT DIRECTLY DOWNLOADABLE — action required",
      "vol": "Unknown",
      "how": "Listed on data.virginia.gov but the resource redirects to the USDOT data inventory. Budget/encumbrance balances are what JE-10 needs and no other public source carries them.",
      "snippet": "# fallback if the extract cannot be obtained:\n# synthesise the encumbrance layer on top of DS-A payments\n# (obligating doc register + encumbrance balances), keeping DS-A amounts real",
      "gotchas": [
        "Treat as a request-and-wait item. Do not block JE-10 on it — the fallback synthesis path is viable and keeps the payment distribution real.",
        "Confirm whether the extract carries entry-level JE source info or only balances; JE-10 needs the former for wrong-document liquidation."
      ],
      "licence": "US federal public domain, subject to the publishing agency's terms."
    }
  },
  "DS-E": {
    "name": "US Standard General Ledger — Accounts and Definitions",
    "org": "Hugging Face (leeroy-jankins)",
    "url": "https://huggingface.co/datasets/leeroy-jankins/US-Standard-General-Ledger-Accounts-And-Definitions",
    "access": "Direct file download from the HF repo (Q&A CSV, chunked-text CSV, corpus TXT, PDF).",
    "fields": "Q&A pairs about USSGL (ID, User, Questions, Agent, Answers) plus the chunked and plain-text corpus of USSGL Bulletin No. 2025-02 and its PDF. NOT a tabular chart of accounts.",
    "why": "Citable USSGL authority text for the judge and for JE-17 classification rationale. A machine-readable account table is not included — if one is needed, extract it from the bulletin text/PDF or from Treasury's TFM USSGL supplement in the phase that needs it (JE-02's validity matrix comes from DS-A's own observed codes regardless).",
    "role": "REFERENCE / MASTER DATA",
    "ingest": {
      "id": "DS-E",
      "status": "FROZEN 2026-08-05 at pinned HF revision (manifest ds_e_2026-08-05)",
      "vol": "Q&A pairs (100 rows) + chunked (1,068 rows) and plain-text corpus of USSGL Bulletin 2025-02 + PDF",
      "how": "Hugging Face datasets: leeroy-jankins/US-Standard-General-Ledger-Accounts-And-Definitions",
      "snippet": "# direct file download pinned to a commit sha — see app/scripts/fetch_ds_e.py",
      "gotchas": [
        "USSGL account numbers will NOT match DS-A's Connecticut account codes (e.g. 51250, 51730). Either build a crosswalk or use USSGL only as the definition authority for JE-17 and generate the validity matrix for JE-02 from DS-A's own observed code set."
      ],
      "licence": "Check the dataset card before redistribution; underlying USSGL content is US federal."
    }
  },
  "DS-F": {
    "name": "SEC Financial Statement Data Sets",
    "org": "US SEC (DERA)",
    "url": "https://www.sec.gov/data-research/sec-markets-data/financial-statement-data-sets",
    "access": "Quarterly ZIPs, 2009Q1–2026Q1. Four tab-delimited UTF-8 files: SUB (submissions, key adsh), NUM (numeric facts, key adsh+tag+version+ddate+qtrs+uom+coreg+segments), PRE (presentation, key adsh+report+line), TAG (taxonomy tags, key tag+version).",
    "fields": "Numeric facts from the face of the financial statements, flattened from XBRL.",
    "why": "Statement-level ground truth for roll-forward and restatement tie-out tasks. Lets a task ask 'does the entry population foot to the reported statement' with a real, checkable answer.",
    "role": "STATEMENT TIE-OUT",
    "ingest": {
      "id": "DS-F",
      "status": "URL confirmed 2026-08-03",
      "vol": "Quarterly ZIPs, 2009Q1–2026Q1, 13KB–121MB each",
      "how": "Index page: https://www.sec.gov/data-research/sec-markets-data/financial-statement-data-sets — quarterly ZIPs each containing four tab-delimited UTF-8 files: SUB (key adsh), NUM (key adsh+tag+version+ddate+qtrs+uom+coreg+segments), PRE (key adsh+report+line), TAG (key tag+version).",
      "snippet": "# SEC REQUIRES a descriptive User-Agent or requests are blocked\ncurl -A 'Centific research contact@centific.com' -O <quarterly-zip-url-from-index-page>",
      "gotchas": [
        "Take the ZIP URLs from the index page rather than assuming a filename pattern — confirm before scripting.",
        "A 'segments' field was added to NUM in December 2024, and SEC has since republished archive quarters with the current schema — every current download shares one 10-column NUM header. Pin snapshots by hash, not by quarter label; 2024q3 + 2026q1 are frozen (manifest ds_f_2026-08-05).",
        "SEC publishes an accuracy disclaimer — these are analysis aids, not a substitute for the filings. Fine for our purposes, worth noting in any external write-up."
      ],
      "licence": "US government public domain; accuracy disclaimer applies."
    }
  },
  "DS-G": {
    "name": "SEC Accounting and Auditing Enforcement Releases (AAERs)",
    "org": "US SEC",
    "url": "https://www.sec.gov/divisions/enforce/friactions.htm",
    "access": "Public releases; AuditFraudBench (arXiv 2606.08345) documents a construction over 84 cases / 295 instances if you want a template.",
    "fields": "Narrative enforcement findings: scheme mechanics, accounts affected, periods, restatement outcome.",
    "why": "Outcome-labelled fraud patterns. Use to source realistic L4 fact patterns with a defensible 'correct answer' anchored in an enforcement finding rather than in an author's opinion.",
    "role": "JUDGMENT FACT PATTERNS",
    "ingest": {
      "id": "DS-G",
      "status": "Public, manual curation required",
      "vol": "AuditFraudBench documents 84 cases / 295 instances as a construction template",
      "how": "SEC AAER releases are public. Select cases whose mechanics map onto a JE population.",
      "snippet": "# no API — curate case narratives, then map each to:\n# {mechanism, accounts, periods, restatement_delta}\n# cross-check restatement_delta against DS-F NUM pre/post values",
      "gotchas": [
        "Case selection needs SME review — not every enforcement narrative reconstructs into a JE population.",
        "Anchor the quantified answer to DS-F values so the ground truth is checkable rather than authored."
      ],
      "licence": "US government public domain."
    }
  },
  "DS-H": {
    "name": "deepAI — accounting anomaly lab data",
    "org": "Schreyer et al. / HSG-AIML",
    "url": "https://github.com/GitiHubi/deepAI",
    "access": "GitHub repo with notebooks and downloadable lab dataset.",
    "fields": "Categorical + numeric JE attributes with injected global and local anomalies.",
    "why": "Reference implementation for how global vs local anomalies are injected and labelled. Borrow the injection taxonomy rather than the data itself.",
    "role": "ANOMALY INJECTION REFERENCE",
    "ingest": {
      "id": "DS-H",
      "status": "URL confirmed",
      "vol": "Lab dataset + notebooks",
      "how": "github.com/GitiHubi/deepAI — borrow the global/local anomaly injection taxonomy, not the data.",
      "snippet": "git clone https://github.com/GitiHubi/deepAI",
      "gotchas": [
        "Use as a reference implementation for injection design. Building scored tasks directly on its data forfeits the real-distribution advantage of DS-A."
      ],
      "licence": "Per repository licence."
    }
  },
  "DS-I": {
    "name": "Kaggle — General Ledger",
    "org": "Kaggle (irfansharif)",
    "url": "https://www.kaggle.com/datasets/irfansharif/generalledger",
    "access": "Kaggle download.",
    "fields": "Small GL extract.",
    "why": "Was reserved as a pre-snapshot smoke-test fixture. The DS-A snapshot exists, so nothing needs it; kept listed only so source ids stay stable.",
    "role": "UNUSED",
    "ingest": {
      "id": "DS-I",
      "status": "Unused — DS-A snapshot exists",
      "vol": "Small",
      "how": "kaggle.com/datasets/irfansharif/generalledger",
      "snippet": "kaggle datasets download -d irfansharif/generalledger",
      "gotchas": [
        "Nothing may depend on this dataset."
      ],
      "licence": "Per Kaggle dataset terms."
    }
  },
  "DS-J": {
    "name": "Synthesised support-document pack",
    "org": "Generated by us",
    "url": "",
    "access": "Generator to be written; one pack per task instance.",
    "fields": "Invoice PDFs/text, contracts and SOW clauses, approval emails, calculation schedules, bank statements, policy manual, vendor master, period-close calendar, prior-period accrual history.",
    "why": "No public source carries preparer support documents — this is the layer that makes L4 possible and is the only place the SME's tacit knowledge gets encoded. Generated deterministically from a seed alongside the defect ledger.",
    "role": "SYNTHESISED EVIDENCE LAYER",
    "ingest": {
      "id": "DS-J",
      "status": "TO BE BUILT — largest engineering item",
      "vol": "One pack per task instance",
      "how": "Deterministic generator seeded from (task_id, seed), emitting the support pack, master data and the defect ledger in a single pass.",
      "snippet": "generate(task_id, seed) -> {\n  ledger_slice,        # sampled from the DS-A snapshot\n  journal_entries,     # postingised, balanced\n  documents[],         # invoices, POs, contracts, emails, schedules\n  master_data{},       # accounts, vendors, periods, approval limits, users\n  policy_manual{},\n  defect_ledger[]      # HELD OUT: entry_id, defect_type, revealing_evidence_id, correct_verdict\n}",
      "gotchas": [
        "The defect ledger must record which evidence reveals each defect — process scoring has no target without it.",
        "Same seed must reproduce byte-identical packs, or cross-model comparisons are not comparable.",
        "L4 packs consume an SME-authored disposition table as input; the generator cannot invent it."
      ],
      "licence": "Ours."
    }
  }
}