Implemented IsFlagged column check

app.py

@@ -56,7 +56,7 @@ FILTER_DOMINANT_COUNTY = False  # process ALL counties (multi-county / full-stat
 #   "csv"   -> Master_Dataset.zip, every result bundled as a CSV (faster: no Excel
 #              writing -- use this for very large / state-sized runs)
 # The individual per-category files are CSV in both modes.
-MASTER_OUTPUT = "csv"
+MASTER_OUTPUT = "excel"
 
 # How records are processed:
 #   "stream" -> ONE COUNTY AT A TIME. The input is split into per-county temp
@@ -189,7 +189,7 @@ def _norm_fips(x):
 # values shaped like "05045-FAULKNER" (FIPS digits, a dash, a county name).
 # Header names are only used as a last-resort fallback.
 FIPS_COUNTY_PATTERN = r"^\s*\d{1,5}\s*-\s*[A-Za-z]"
-FALLBACK_FIPS_HEADERS = ["** REC FIPS-COUNTY NAME", "** REC FIPS-COUNTY", "County", "DocumentRecordingCountyFIPS"]
+FALLBACK_FIPS_HEADERS = ["** REC FIPS-COUNTY NAME", "** REC FIPS-COUNTY", "County"]
 
 
 def detect_fips_county_column(df, sample_size=1000, min_score=0.5):
@@ -258,6 +258,22 @@ def ensure_combined_county(df, fips_col, reason, wl, log=None):
     return "County", f"synthesized '<FIPS>-<NAME>' from bare FIPS column '{fips_col}'"
 
 
+def ensure_flag_columns(df, log=None):
+    """The Backoffice upload keys on RecordKey + IsFlagged + ComesFromMtg2, so the
+    final files must carry those flag columns. Data pulled straight from SQL lacks
+    IsFlagged / ComesFromMtg2 (the PBIX added them), so create them as FALSE when
+    missing. Columns already present (e.g. a PBIX pull) are left untouched."""
+    added = []
+    for col in ("IsFlagged", "ComesFromMtg2"):
+        if col not in df.columns:
+            df[col] = False
+            added.append(col)
+    if added and log is not None:
+        log.append(f"Added missing flag column(s) {', '.join(added)} = FALSE "
+                   f"(not in input; required for the Backoffice upload).")
+    return df
+
+
 # =====================================================================
 # SELF-TEST  (startup health check — same logic as the notebook's cell)
 # =====================================================================
@@ -657,6 +673,7 @@ def run_pipeline(uploaded_files, reference_file, blank_cities_not_in_list,
     del dfs
     gc.collect()
     _step(log, f"Merged shape: {df.shape}")
+    ensure_flag_columns(df, log)
 
     # ---- Locate the FIPS-county column by CONTENT --------------------
     fips_col, fips_reason = detect_fips_county_column(df)
@@ -1046,6 +1063,7 @@ def process_county(cdf, wl, fips_col, cluster_offset,
     Cluster_IDs are offset by `cluster_offset` so, processing counties in sorted
     order, the numbering matches the whole-dataset ("memory") path."""
     ignore = []
+    ensure_flag_columns(cdf, None)
     fips_col, _r = ensure_combined_county(cdf, fips_col, "", wl, None)
     cdf, audit = fips_validate(cdf, blank_cities_not_in_list, ignore, wl=wl, fips_col=fips_col)
 
@@ -1221,9 +1239,10 @@ def _split_by_county(uploaded_files, out_dir, log, chunksize=STREAM_CHUNK_ROWS):
     else:
         sample = pd.read_excel(first)
     fips_col, reason = detect_fips_county_column(sample)
+    sample_cols = list(sample.columns)
     del sample
     if fips_col is None:
-        return None, reason, []
+        return None, reason, [], sample_cols
 
     county_paths = {}       # county label -> temp path
     counter = [0]
@@ -1255,7 +1274,7 @@ def _split_by_county(uploaded_files, out_dir, log, chunksize=STREAM_CHUNK_ROWS):
             _step(log, f"Split: routed {total:,} rows into {len(county_paths)} counties")
 
     ordered = sorted(county_paths.items(), key=lambda kv: str(kv[0]))
-    return fips_col, reason, ordered
+    return fips_col, reason, ordered, sample_cols
 
 
 def run_streaming(uploaded_files, reference_file, blank_cities_not_in_list,
@@ -1277,11 +1296,15 @@ def run_streaming(uploaded_files, reference_file, blank_cities_not_in_list,
         return "\n".join(log) + f"\n\nCould not parse reference list: {e}", []
 
     _step(log, "Splitting input by county (streaming)...")
-    fips_col, reason, counties = _split_by_county(uploaded_files, out_dir, log)
+    fips_col, reason, counties, sample_cols = _split_by_county(uploaded_files, out_dir, log)
     if fips_col is None:
         return ("\n".join(log) + "\n\nCOLUMN ERROR: could not find a FIPS-county column. "
                 "Expected values like '05045-FAULKNER'."), []
     _step(log, f"FIPS-county column: '{fips_col}' ({reason}) | {len(counties)} counties")
+    _missing_flags = [c for c in ("IsFlagged", "ComesFromMtg2") if c not in sample_cols]
+    if _missing_flags:
+        _step(log, f"Adding missing flag column(s) {', '.join(_missing_flags)} = FALSE "
+                   f"(required for the Backoffice upload)")
 
     # combined output writers
     audit_path = out_dir / "FIPS_Validation_Audit_ChangedRecords.csv"