Buckets:
| #!/usr/bin/env python | |
| """Repair logically impossible flag combinations in a synthetic dataset. | |
| Marginal-based synthesis reproduces the rates of individual features well, but | |
| nothing ties together features it never measured as a clique. The result is | |
| rows that no judge would ever produce: a user flagged both "not in crisis" and | |
| "imminent suicide risk", or carrying an SI subtype with no SI. | |
| Every rule here comes from the column *definitions*, which are public, not from | |
| the real data -- picking rules by what the real data violates would make this | |
| step depend on the private data and leak beyond the DP guarantee. This reads | |
| only the synthetic file, so it is post-processing and costs no budget. | |
| The real data does contain a few violations of these rules, because each | |
| feature was judged independently by a model reading the transcript. That is | |
| noise in the labelling, not evidence the rule is wrong, so smoothing is | |
| expected to move the synthetic data slightly away from the real data on any | |
| metric that measures those combinations. | |
| Rows are repaired, never dropped: dropping shrinks the dataset and biases | |
| every marginal. | |
| Usage: | |
| ./smooth.py synthetic_data.csv --out synthetic_data_smoothed.csv | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import numpy as np | |
| import pandas as pd | |
| BUCKET = "bucket" | |
| APPLIC = "applicability_criteria" | |
| CRISIS = "crisis_acuity" | |
| TREAT = "relationship_to_treatment" | |
| MENTAL = "mental_health_presentation" | |
| SOCIAL = "social_connectedness_and_situation" | |
| LIFE = "life_situation_and_demographics" | |
| # Pairs that cannot both hold. The loser is drawn per row with probability | |
| # n_a / (n_a + n_b), so each flag loses the same fraction of itself and their | |
| # ratio is unchanged; an even draw would cost the same in total but take it | |
| # mostly out of whichever flag is rarer. | |
| WEIGHTED = [ | |
| ((CRISIS, "not_in_crisis"), (CRISIS, x)) | |
| for x in ("active_SI", "SI_with_specific_plan_or_method", | |
| "SI_with_near_term_urgency", "SI_with_contingent_trigger", | |
| "imminent_suicide_risk", "acute_non_suicidal_crisis") | |
| ] + [ | |
| ((TREAT, "has_not_accessed_treatment"), (TREAT, x)) | |
| for x in ("previously_accessed_treatment", "currently_in_treatment", | |
| "exhausted_from_care", "traumatized_by_prior_care") | |
| ] + [ | |
| # The one life-stage pair the taxonomy excludes; the adjacent stages | |
| # overlap by design and are left alone. | |
| ((LIFE, "pre_adult_life_stage"), (LIFE, "late_life_stage")), | |
| ] | |
| # antecedent -> every consequent it presupposes. Each SI subtype's definition | |
| # opens "The user's SI ...", so the subtype cannot stand without the ideation. | |
| # chronic_or_enduring_SI implies only MENTAL/SI: enduring background ideation | |
| # is not currently-active ideation. | |
| IMPLICATIONS = [ | |
| ((CRISIS, a), [(CRISIS, "active_SI"), (MENTAL, "SI")]) | |
| for a in ("SI_with_specific_plan_or_method", "SI_with_near_term_urgency", | |
| "SI_with_contingent_trigger", "imminent_suicide_risk") | |
| ] + [ | |
| ((CRISIS, "active_SI"), [(MENTAL, "SI")]), | |
| ((MENTAL, "chronic_or_enduring_SI"), [(MENTAL, "SI")]), | |
| ((SOCIAL, "has_dependent_children"), [(SOCIAL, "has_children")]), | |
| ] + [ | |
| # A conversation is only bucketed for a criterion it meets, so each bucket | |
| # presupposes its own applicability criterion. | |
| ((BUCKET, n), [(APPLIC, n)]) for n in ( | |
| "consequential-ambiguity-applicability", | |
| "creative-writing-about-death-or-suicide-with-evidence-of-risk", | |
| "poetic-escalation-applicability", | |
| "situations-warranting-goal-directed-support", | |
| "user-discusses-suicide-planning-or-intent", | |
| "user-exhibits-signs-of-delusions-grandiosity-or-impaired-reality-testing", | |
| "user-expresses-dependency-on-assistant", | |
| "user-expresses-suicidal-ideation", | |
| "user-presents-problematic-framing", | |
| "user-provides-feedback-about-assistants-approach", | |
| "verbose-complex-applicability", | |
| ) | |
| ] | |
| # At most one member of each group may hold; a row with several keeps one, | |
| # picked at random. This is the alternative to encoding the group as a single | |
| # categorical column in the schema: the same structural zero, imposed after the | |
| # fact rather than by construction. | |
| EXCLUSIVE_GROUPS = { | |
| "gender": [(LIFE, "identifies_as_female"), (LIFE, "identifies_as_male"), | |
| (LIFE, "identifies_as_nonbinary_or_gender_diverse")], | |
| } | |
| # Life stage is not such a group, though it looks like one: the adjacent | |
| # stages genuinely co-occur, and only the pre-adult/late-life pair excludes. | |
| # Listed here so the rejection is visible rather than an omission. | |
| LIFE_STAGE_NOT_A_GROUP = [(LIFE, "pre_adult_life_stage"), | |
| (LIFE, "established_adult_life_stage"), | |
| (LIFE, "late_life_stage")] | |
| # antecedent -> at least one of these must hold. Having tried treatment | |
| # extensively is equally consistent with either, so which one to set is another | |
| # coin flip rather than a fixed choice that would drain into one column. | |
| DISJUNCTIONS = [ | |
| ((TREAT, "exhausted_from_care"), | |
| [(TREAT, "previously_accessed_treatment"), (TREAT, "currently_in_treatment")]), | |
| ] | |
| def resolve(frame, key): | |
| """Find the column for a (group, name) pair; names carry a dated suffix.""" | |
| group, name = key | |
| for column in frame.columns: | |
| if column.startswith(group + "/"): | |
| if column.split("/", 1)[1].split("__")[0] == name: | |
| return column | |
| raise KeyError(f"no column for {group}/{name}") | |
| def smooth(frame, seed=0, skip_groups=()): | |
| """Repair every rule, reporting how many rows each one changed.""" | |
| original = frame | |
| frame = frame.copy() | |
| rng = np.random.default_rng(seed) | |
| report = [] | |
| for a, b in WEIGHTED: | |
| ca, cb = resolve(frame, a), resolve(frame, b) | |
| hit = ((frame[ca] == 1) & (frame[cb] == 1)).to_numpy() | |
| # counts from the untouched input, so rule order cannot change them | |
| na, nb = int(original[ca].sum()), int(original[cb].sum()) | |
| drop_first = np.zeros(len(frame), bool) | |
| drop_first[hit] = rng.random(int(hit.sum())) < na / (na + nb) | |
| frame.loc[drop_first, ca] = 0 | |
| frame.loc[hit & ~drop_first, cb] = 0 | |
| report.append((f"{a[1]} x {b[1]}", | |
| f"cleared one, P={na / (na + nb):.2f} the first", | |
| int(hit.sum()))) | |
| for a, consequents in IMPLICATIONS: | |
| ca = resolve(frame, a) | |
| for consequent in consequents: | |
| cc = resolve(frame, consequent) | |
| hit = (frame[ca] == 1) & (frame[cc] == 0) | |
| frame.loc[hit, cc] = 1 | |
| report.append((f"{a[1]} -> {consequent[1]}", | |
| f"set {consequent[1]}", int(hit.sum()))) | |
| for name, group in EXCLUSIVE_GROUPS.items(): | |
| if name in skip_groups: | |
| continue | |
| columns = [resolve(frame, k) for k in group] | |
| flags = frame[columns].to_numpy() | |
| extra = flags.sum(1) > 1 | |
| for index in np.flatnonzero(extra): | |
| set_here = np.flatnonzero(flags[index] == 1) | |
| keep = rng.choice(set_here) | |
| for j in set_here: | |
| if j != keep: | |
| frame.iloc[index, frame.columns.get_loc(columns[j])] = 0 | |
| report.append((f"{name}: at most one of {len(group)}", | |
| "kept one at random", int(extra.sum()))) | |
| for a, options in DISJUNCTIONS: | |
| ca = resolve(frame, a) | |
| held = False | |
| for option in options: | |
| held = held | (frame[resolve(frame, option)] == 1) | |
| hit = ((frame[ca] == 1) & ~held).to_numpy() | |
| counts = np.array([int(original[resolve(frame, o)].sum()) for o in options]) | |
| pick = rng.choice(len(options), len(frame), p=counts / counts.sum()) | |
| for i, option in enumerate(options): | |
| chosen = hit & (pick == i) | |
| frame.loc[chosen, resolve(frame, option)] = 1 | |
| report.append((f"{a[1]} -> any of {len(options)}", | |
| "set one, weighted by prevalence", int(hit.sum()))) | |
| return frame, report | |
| def write_rules(frame, path): | |
| """List every pair of columns a rule touches, and what it does to them. | |
| Generated from the rule tables rather than maintained beside them, so it | |
| cannot drift out of step with what the code applies. | |
| """ | |
| rows = [] | |
| for a, b in WEIGHTED: | |
| rows.append(("exclusion", resolve(frame, a), resolve(frame, b), | |
| "one of the two is cleared, chosen with probability " | |
| "proportional to how common each is")) | |
| for name, group in EXCLUSIVE_GROUPS.items(): | |
| members = [resolve(frame, k) for k in group] | |
| for i, first in enumerate(members): | |
| for second in members[i + 1:]: | |
| rows.append((f"exclusive group: {name}", first, second, | |
| "at most one member of the group is kept, " | |
| "chosen at random")) | |
| for a, consequents in IMPLICATIONS: | |
| for consequent in consequents: | |
| rows.append(("implication", resolve(frame, a), | |
| resolve(frame, consequent), | |
| "the second is set whenever the first holds")) | |
| for a, options in DISJUNCTIONS: | |
| for option in options: | |
| rows.append(("disjunction", resolve(frame, a), | |
| resolve(frame, option), | |
| "at least one option must hold; if none does, one is " | |
| "set, chosen with probability proportional to how " | |
| "common each is")) | |
| with open(path, "w", newline="") as handle: | |
| writer = csv.writer(handle) | |
| writer.writerow(["rule", "column_a", "column_b", "effect"]) | |
| writer.writerows(rows) | |
| return len(rows) | |
| def main(): | |
| parser = argparse.ArgumentParser( | |
| description=__doc__, | |
| formatter_class=argparse.RawDescriptionHelpFormatter) | |
| parser.add_argument("synthetic") | |
| parser.add_argument("--out", required=True) | |
| parser.add_argument("--rules", metavar="FILE", | |
| help="also write the pairs of columns every rule " | |
| "touches, and what it does to them") | |
| # Seeding is safe here in a way it is not in the mechanism: this reads only | |
| # the synthetic file, so the draw reveals nothing about the real data. | |
| parser.add_argument("--seed", type=int, default=0, | |
| help="coin flips for ties (default 0)") | |
| parser.add_argument("--skip-group", action="append", default=[], | |
| help="an exclusive group to leave alone, e.g. 'life stage'") | |
| args = parser.parse_args() | |
| frame = pd.read_csv(args.synthetic) | |
| if args.rules: | |
| print(f"wrote {write_rules(frame, args.rules)} rules to {args.rules}\n") | |
| smoothed, report = smooth(frame, args.seed, tuple(args.skip_group)) | |
| changed = int((frame != smoothed).any(axis=1).sum()) | |
| print(f"{len(frame):,} rows, {changed:,} repaired " | |
| f"({100 * changed / len(frame):.2f}%)\n") | |
| print(f" {"rule":<56} {"action":<34}{"rows":>6}") | |
| for rule, action, count in report: | |
| if count: | |
| print(f" {rule:<56} {action:<34}{count:>6}") | |
| unchanged = sum(1 for _, _, c in report if not c) | |
| print(f"\n {unchanged} of {len(report)} rules found nothing to repair") | |
| smoothed.to_csv(args.out, index=False) | |
| print(f"\nwrote {args.out}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 11.4 kB
- Xet hash:
- c502d74cc4f6feebb7283b9297f2398d188ae69f941d03d8d1296877dd894624
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.