configs/data.yaml

# ---------------------------------------------------------------------------
# Dataset definition, batch structure, continuation joins and exclusions.
#
# PROVISIONAL VALUES: every block carrying `verified: false` records the
# conventionally used value from the source release, NOT a value this project
# has confirmed against the actual files. Phase 2 must confirm each against the
# parsed data, flip the flag, and record the evidence in
# docs/01_dataset_and_eda.md. Cell counts and exclusion lists that are merely
# inherited are exactly the kind of unchecked assumption that produces a
# reconciliation table nobody can defend.
# ---------------------------------------------------------------------------

dataset:
  name: "Severson-Attia LFP/graphite fast-charge cycling corpus"
  chemistry: "LFP / graphite"
  cell_model: "A123 APR18650M1A"
  nominal_capacity_ah: 1.1
  nominal_voltage_v: 3.3
  cycling_temperature_c: 30
  discharge_protocol: "identical 4C discharge to 2.0 V"
  # End-of-life definition. Cycle life = first cycle at which discharge
  # capacity falls below this fraction of nominal capacity.
  # Phase 2 RECOMPUTES cycle life from the capacity series using this
  # definition and compares against any precomputed field. A precomputed label
  # is not trusted; every disagreement is reported.
  eol_capacity_fraction: 0.80

source:
  # TASK 2.0 COMPLETE. Each URL below was verified by request, not recalled:
  # the server's own Content-Disposition header supplied the filename and
  # Content-Length the exact size. The first 19 bytes of each file were read
  # via a Range request and confirmed to be the ASCII string
  # "MATLAB 7.3 MAT-file", which is why h5py is required and scipy.io cannot
  # be used.
  #
  # The host sends no Accept-Ranges header but DOES honour Range requests,
  # answering HTTP 206 with a correct Content-Range. Resume is therefore
  # implemented in src/data/downloader.py on empirically confirmed behaviour
  # rather than on an advertised capability.
  #
  # `verified` stays false until sha256 digests are computed from the
  # downloaded bytes; the schema in src/utils/config.py refuses to accept
  # verified: true while any digest is missing.
  verified: true
  verified_on: "2026-08-03"
  landing_page: "https://data.matr.io/1/projects/5c48dd2bc625d700019f3204"
  primary_citation: >-
    Severson, K.A. et al. Data-driven prediction of battery cycle life before
    capacity degradation. Nature Energy 4, 383-391 (2019).
  extension_citation: >-
    Attia, P.M. et al. Closed-loop optimization of fast-charging protocols for
    batteries with machine learning. Nature 578, 397-402 (2020).
  files:
    - name: "2017-05-12_batchdata_updated_struct_errorcorrect.mat"
      url: "https://data.matr.io/1/api/v1/file/5c86c0b5fa2ede00015ddf66/download"
      size_bytes: 3025320241
      batch: "batch1"
      sha256: 9d928ab978f0e3c70b31cb833a749fedd35094d01af76475d69b40aa3497f5ba
    - name: "2017-06-30_batchdata_updated_struct_errorcorrect.mat"
      url: "https://data.matr.io/1/api/v1/file/5c86bf13fa2ede00015ddd82/download"
      size_bytes: 2007331155
      batch: "batch2"
      sha256: 63ab200d09ecb237fee5ef3a5c5db76e3212e3206a0bd92f769e1427fed338b8
    - name: "2018-04-12_batchdata_updated_struct_errorcorrect.mat"
      url: "https://data.matr.io/1/api/v1/file/5c86bd64fa2ede00015ddbb2/download"
      size_bytes: 3236690412
      batch: "batch3"
      sha256: 62c30e413b63e6144720e016deed3661fac8468641794a5807b123fe84717998
  total_size_bytes: 8269341808
  format: "MATLAB v7.3 (HDF5) - must be read with h5py, NOT scipy.io"
  # The Attia et al. (2020) extension (batch 4 onward, taking the corpus to
  # 169 cells) lives in a DIFFERENT matr.io project. Candidate file IDs probed
  # during Task 2.0 returned zero-length responses, so no URL is recorded here.
  # Phase 2 targets the 124-cell primary corpus; the extension is deferred
  # rather than guessed at.
  extension_located: false

paths:
  raw: "data/raw"
  interim: "data/interim"
  processed: "data/processed"
  cells: "data/interim/cells"

# ---------------------------------------------------------------------------
# Batch structure. Batch identity is treated as a proxy for production
# campaign; see docs/01_dataset_and_eda.md for the manufacturing-analogue
# mapping and its honest limitations.
# ---------------------------------------------------------------------------
batches:
  batch1:
    date: "2017-05-12"
    role: "in_distribution"
    expected_cells_raw: 46
    expected_cells_used: 41
    # VERIFIED in Phase 2: parser found exactly 46 records; 5 excluded; 41 used.
    verified: true
    note: "Five cells continue into batch 2 and are joined, not double-counted."
  batch2:
    date: "2017-06-30"
    role: "in_distribution"
    expected_cells_raw: 48
    expected_cells_used: 43
    # VERIFIED in Phase 2: 48 records found; 5 consumed by continuation joins; 43 used.
    verified: true
    note: >-
      Five records are continuations of batch-1 cells and are consumed by the
      join, so they are not counted as independent cells here.
  batch3:
    date: "2018-04-12"
    role: "out_of_distribution"
    expected_cells_raw: 46
    expected_cells_used: 40
    # VERIFIED in Phase 2: 46 records found; 6 excluded; 40 used.
    verified: true
    note: >-
      Generated later under different conditions. This is the natural
      out-of-distribution split for RQ4 (process-recipe / campaign shift).

# ---------------------------------------------------------------------------
# Batch-1 cells whose cycling continued as a batch-2 record. These must be
# JOINED into a single physical cell, never counted twice.
#
# WHY this matters beyond bookkeeping: a cell split across two records has its
# true cycle life truncated in the batch-1 record. Failing to join would inject
# five artificially short-lived cells into training and corrupt the label.
#
# `known_extra_cycles` records the continuation length cited in the source
# release. Phase 2 must RECOMPUTE the joined length from the data and treat any
# disagreement as a parser bug, not as a value to overwrite.
# ---------------------------------------------------------------------------
continuations:
  - {primary: "b1c0", continued_as: "b2c7",  known_extra_cycles: 662, verified: true}
  - {primary: "b1c1", continued_as: "b2c8",  known_extra_cycles: 981, verified: true}
  - {primary: "b1c2", continued_as: "b2c9",  known_extra_cycles: 1060, verified: true}
  - {primary: "b1c3", continued_as: "b2c15", known_extra_cycles: 208, verified: true}
  - {primary: "b1c4", continued_as: "b2c16", known_extra_cycles: 482, verified: true}

# ---------------------------------------------------------------------------
# Exclusion list. EVERY excluded cell carries an ID and a reason -- the schema
# in src/utils/config.py rejects an entry without one, which makes the project
# rule "never silently drop a cell" structurally enforced rather than a matter
# of discipline.
#
# `verified: false` means: this exclusion is inherited from the source release
# and Phase 2 must independently confirm the stated reason holds in the parsed
# data (e.g. actually confirm the cell never reaches the 80% threshold) before
# the cell is dropped from the analysis.
# ---------------------------------------------------------------------------
exclusions:
  - {cell_id: "b1c8", batch: "batch1", reason: "Right-censored: discharge capacity never reaches the end-of-life threshold across 878 cycles, so no cycle-life label exists.", verified: true}
  - {cell_id: "b1c10", batch: "batch1", reason: "Right-censored: discharge capacity never reaches the end-of-life threshold across 905 cycles, so no cycle-life label exists.", verified: true}
  - {cell_id: "b1c12", batch: "batch1", reason: "Right-censored: discharge capacity never reaches the end-of-life threshold across 901 cycles, so no cycle-life label exists.", verified: true}
  - {cell_id: "b1c13", batch: "batch1", reason: "Right-censored: discharge capacity never reaches the end-of-life threshold across 896 cycles, so no cycle-life label exists.", verified: true}
  - {cell_id: "b1c22", batch: "batch1", reason: "Right-censored: discharge capacity never reaches the end-of-life threshold across 890 cycles, so no cycle-life label exists.", verified: true}
  - {cell_id: "b3c2", batch: "batch3", reason: "Noisy channel, independently confirmed: trajectory roughness ranks 3rd of 46 in batch 3, above the 90th percentile of its peers.", verified: true}
  - {cell_id: "b3c23", batch: "batch3", reason: "Noisy channel, independently confirmed: roughest trajectory of all 46 batch-3 cells, and additionally right-censored across 2189 cycles.", verified: true}
  - {cell_id: "b3c32", batch: "batch3", reason: "Right-censored across 2237 cycles, so no cycle-life label exists. NOTE: the source release cites a noisy channel, which this project could NOT confirm (roughness ranks only 15 of 46); the censoring is the defensible reason.", verified: true}
  - {cell_id: "b3c37", batch: "batch3", reason: "Excluded by the source release as a noisy channel. NOT INDEPENDENTLY CONFIRMED: roughness ranks 20 of 46 and the cell yields a valid cycle life of 1389. Retained in the exclusion list only for comparability with the standard 124-cell corpus.", verified: false}
  - {cell_id: "b3c42", batch: "batch3", reason: "Excluded by the source release as a noisy channel. NOT INDEPENDENTLY CONFIRMED: roughness ranks 22 of 46 and the cell yields a valid cycle life of 1641. Retained in the exclusion list only for comparability with the standard 124-cell corpus.", verified: false}
  - {cell_id: "b3c43", batch: "batch3", reason: "Excluded by the source release as a noisy channel. NOT INDEPENDENTLY CONFIRMED: roughness ranks 23 of 46 and the cell yields a valid cycle life of 1045. Retained in the exclusion list only for comparability with the standard 124-cell corpus.", verified: false}

reconciliation:
  # Phase 2 prints a table: found / joined / excluded / used, and asserts the
  # final count matches. 124 cells is the figure reported for the primary
  # dataset; the 45-cell Attia extension would bring the corpus to 169.
  expected_cells_used: 124
  expected_cells_used_with_extension: 169
  # VERIFIED in Phase 2: 140 raw records -> 5 consumed by joins -> 135 physical
  # cells -> 11 excluded -> exactly 124 used. Every per-batch count matches too.
  # The 169-cell figure remains unverified: the Attia extension is not located.
  verified: true

# ---------------------------------------------------------------------------
# Parsing and storage policy (Phase 2).
# ---------------------------------------------------------------------------
parsing:
  # Summary series (one row per cycle: capacity, IR, temperature, charge time)
  # are retained for EVERY cycle. Cycle life, capacity-fade trajectories and
  # resistance evolution all need the full trajectory to end-of-life.
  #
  # Within-cycle traces (V, I, Q, T, t and the interpolated Q(V) family) are
  # retained only up to this cycle. WHY this is methodologically safe rather
  # than a shortcut: the leakage rule caps every feature at budget N <= 100, so
  # no feature may legitimately read a within-cycle trace beyond cycle 100, and
  # the Phase 3 figures that use within-cycle data (Q(V) at cycles 10 and 100,
  # the DeltaQ(V) curve family) also stop at 100. Retaining all ~96,700 cycles
  # of raw traces would cost several GiB to store data the project's own rules
  # forbid using.
  #
  # The value carries 20 cycles of headroom above the maximum budget so that a
  # budget of exactly 100 has room for baseline/late cycle pairs.
  within_cycle_max_cycle: 120
  # Written with zstd: these frames are large and read many times downstream.
  parquet_compression: "zstd"

# ---------------------------------------------------------------------------
# Validator bounds (Phase 2). These checks FAIL LOUDLY -- they do not warn and
# continue. A physically impossible value means the parser misread the file
# layout, and every downstream number would be fiction.
# ---------------------------------------------------------------------------
validation:
  # 0 < Q <= 1.2 * nominal. The 1.2 headroom accommodates genuine early-life
  # capacity above nameplate without admitting a unit-conversion error.
  capacity_max_fraction_of_nominal: 1.2

  # The CHARGING PROTOCOL setpoints are a 2.0 V discharge cutoff and a 3.6 V
  # charge cutoff. Measured samples legitimately sit outside them: the cycler
  # samples at a finite rate, so the last sample of a discharge overshoots the
  # cutoff before the step terminates. Measured across the corpus, voltage
  # spans [1.919, 3.614] V.
  #
  # The validation bound is therefore widened past the protocol setpoints.
  # This does NOT weaken the check: its purpose is to catch a parser that
  # misread the file layout, and such an error produces zeros, negatives, or
  # values orders of magnitude off -- not an 80 mV cutoff overshoot. Recording
  # both values keeps the distinction explicit.
  protocol_voltage_range_v: [2.0, 3.6]
  voltage_range_v: [1.85, 3.70]

  # Cells were cycled in a 30 degC chamber. Measured span across the corpus is
  # [27.3, 40.4] degC (cells self-heat under 4C discharge), comfortably inside
  # this bound, which is left at its original value.
  temperature_range_c: [20, 50]
  require_monotonic_cycle_index: true
  forbid_duplicate_cycle_index: true
  # Compare recomputed cycle life against any precomputed field and report
  # every disagreement larger than this many cycles.
  cycle_life_disagreement_tolerance: 0

  # Cycle life is RECOMPUTED from the discharge-capacity series; the
  # precomputed field in the .mat file is treated as a claim to be checked,
  # never as the label. WHY: the entire target variable of the project depends
  # on this definition, and inheriting it unexamined would make every
  # downstream number rest on an assumption nobody verified.
  cycle_life_recompute:
    # Cycle at which discharge capacity first falls to eol_capacity_fraction
    # of nominal (0.80 * 1.1 Ah = 0.88 Ah).
    method: "first_below_threshold"
    # Rolling-median window applied before thresholding. Measured capacity is
    # noisy cycle-to-cycle, and a single spurious dip would otherwise truncate
    # a cell's life by hundreds of cycles. 1 disables smoothing.
    smoothing_window: 5
    # Ignore threshold crossings before this cycle. Early-life measurement
    # artefacts occasionally produce an implausible first-cycle reading.
    min_cycle: 10

    # ---------------------------------------------------------------------
    # EMPIRICALLY DERIVED, NOT ASSUMED. Measured in Phase 2 across all 46
    # batch-1 cells (see docs/01_dataset_and_eda.md).
    #
    # These cells were cycled UNTIL they reached 80% of nominal, at which
    # point the experiment stopped. Consequently capacity never actually
    # crosses below 0.88 Ah -- the record simply ends just above it. A naive
    # "first cycle strictly below 0.88 Ah" rule therefore classifies EVERY
    # cell as never reaching end of life, which is plainly wrong.
    #
    # The observed minima separate cleanly into two groups:
    #   36 cells that reached end of life : min QD in [0.8801, 0.8834] Ah
    #   10 cells that did not             : min QD in [0.9131, 1.0433] Ah
    # (the 10 being the 5 continuation cells, whose records are truncated
    # because cycling continues in batch 2, and the 5 excluded cells).
    #
    # A cell is treated as having reached end of life if its capacity comes
    # within this fraction of the threshold.
    #
    # CALIBRATED IN SMOOTHED SPACE, which is where the decision is actually
    # made. The rolling median lifts a cell's minimum slightly, because a
    # monotonically fading series has a trailing-median above its final value,
    # so a tolerance calibrated on raw minima wrongly censors cells that fade
    # to just above threshold. Measured separation of smoothed minima:
    #   window 1: EOL max 0.8834 | non-EOL min 0.9131
    #   window 3: EOL max 0.8846 | non-EOL min 0.9142
    #   window 5: EOL max 0.8859 | non-EOL min 0.9150   <- window in use
    # The gap is ~0.029 Ah at every window. 0.02 puts the effective threshold
    # at 0.8976 Ah (81.6% of nominal), mid-gap, and classifies all 46 batch-1
    # cells correctly at ALL three smoothing windows -- so the result does not
    # depend on the smoothing choice.
    #
    # Calibrated on batch 1; re-verified against batches 2 and 3 in Phase 2.
    eol_tolerance_fraction: 0.02

  # ---------------------------------------------------------------------
  # PLACEHOLDER ROW -- A REAL STRUCTURAL DIFFERENCE BETWEEN BATCHES,
  # measured in Phase 2, not assumed.
  #
  #   batch1: ALL 46 cells begin with a row in which every summary field is
  #           exactly zero. It is a structural artefact, not a measured cycle,
  #           and is why the source literature takes "discharge capacity at
  #           cycle 2" as the first usable value.
  #   batch2: 0 of 43 cells have one.
  #   batch3: 0 of 46 cells have one.
  #
  # CONSEQUENCE, and it matters for Phase 4: raw cycle numbering does NOT mean
  # the same thing across batches. Cycle 2 is the first measured cycle in
  # batch 1 but the second in batches 2 and 3. Slicing "cycles 1..N" on the raw
  # index would therefore give batch-1 cells one fewer measured cycle than the
  # others at every budget -- a systematic, batch-correlated bias in exactly
  # the variable RQ4 tests for.
  #
  # RESOLUTION: the placeholder row is RETAINED in the stored parquet, so the
  # data stays faithful to the source and cycle numbering remains comparable
  # with published work. A separate `cycle_measured` column numbers only
  # measured cycles from 1, consistently across every batch, and that is what
  # feature budgets slice on.
  placeholder_row:
    expect_at_most_one: true
    expect_at_first_cycle: true
    batches_with_placeholder: ["batch1"]

  # ---------------------------------------------------------------------
  # DISCRIMINATING A PARSER BUG FROM AN INSTRUMENT ARTEFACT.
  #
  # These are different failures and only one invalidates the pipeline:
  #   * A parser bug is SYSTEMATIC ACROSS CELLS. Misreading the file layout
  #     corrupts every cell the same way.
  #   * An instrument artefact is LOCALISED TO A CELL. Measured here: 4 of 135
  #     cells (3.0%) contain any implausible capacity reading, and 3 of those
  #     are a single cycle. The fourth, b1c18, shows q_charge ramping
  #     monotonically 1.335 -> 1.968 Ah across cycles 16-40 while q_discharge
  #     stays flat at 1.070 Ah, which is a coulomb-counter reset failure on the
  #     charge channel of one cell -- not a mis-parse, which could not leave
  #     the discharge channel of the same cell intact.
  #
  # So the run FAILS only if the fraction of AFFECTED CELLS exceeds
  # systematic_cell_fraction. Per-cell violations are reported as WARN and the
  # cell is flagged in the cohort so Phase 4 handles it deliberately rather
  # than inheriting it silently.
  systematic_violation_fraction: 0.01   # within a cell: escalates the message
  systematic_cell_fraction: 0.20        # across cells: fails the run

  # ---------------------------------------------------------------------
  # MAXIMUM PLAUSIBLE DURATION OF ONE CYCLE, in minutes. Used by the Phase 4
  # feature builder to mask cycles whose recorded elapsed time cannot be real.
  #
  # DERIVED FROM THE DATA, measured across all 135 cells over cycles 1-100:
  #   median cycle duration, median across cells : 51.8 min
  #   118 of 135 cells never exceed              : 78.6 min
  #   16 cells show ONE isolated long cycle      : ~460 min (plausibly a real
  #                                                rest, and 1 cycle in 100)
  #   b1c18 has a MEDIAN cycle duration of       : 2236 min (37 hours)
  #
  # These cells are charged at 3.6-8C and discharged at 4C, so a genuine cycle
  # takes tens of minutes. A cell whose TYPICAL cycle reads 37 hours has a
  # broken time channel, not a long experiment -- and because thermal exposure
  # is an integral over time, that error propagates directly into Group D:
  # b1c18's thermal_exposure came out 71x the cohort median, which alone drove a
  # cross-validated ridge prediction to -50.9 on a target of 2.84.
  #
  # 300 minutes sits ~6x above the median and ~4x above the ceiling of the 118
  # clean cells, so it masks the broken channel without touching normal cycling.
  # Applied per CYCLE, not per cell: a cell with one long rest keeps its other
  # 99 cycles.
  max_cycle_duration_minutes: 300