Implemented percentage and RURAL vs URBAN columns to summary.

app.py

@@ -74,6 +74,83 @@ STREAM_CHUNK_ROWS = 400_000   # rows per chunk while splitting the input by coun
 # on big states, so it's OFF by default. Set True if you actually need that file.
 WRITE_RAWFILE = False
 
+# Threshold for the RURAL/URBAN label (records <= this -> RURAL, else URBAN).
+RURAL_MAX_RECORDS = 10000
+
+# State FIPS prefix (first 2 digits of a county FIPS) -> USPS abbreviation, so the
+# Summary can fill the tracking file's "State" column with no manual lookup.
+STATE_FIPS_ABBR = {
+    "01": "AL", "02": "AK", "04": "AZ", "05": "AR", "06": "CA", "08": "CO",
+    "09": "CT", "10": "DE", "11": "DC", "12": "FL", "13": "GA", "15": "HI",
+    "16": "ID", "17": "IL", "18": "IN", "19": "IA", "20": "KS", "21": "KY",
+    "22": "LA", "23": "ME", "24": "MD", "25": "MA", "26": "MI", "27": "MN",
+    "28": "MS", "29": "MO", "30": "MT", "31": "NE", "32": "NV", "33": "NH",
+    "34": "NJ", "35": "NM", "36": "NY", "37": "NC", "38": "ND", "39": "OH",
+    "40": "OK", "41": "OR", "42": "PA", "44": "RI", "45": "SC", "46": "SD",
+    "47": "TN", "48": "TX", "49": "UT", "50": "VT", "51": "VA", "53": "WA",
+    "54": "WV", "55": "WI", "56": "WY",
+}
+
+# Exact column layout of the tracking file's Main sheet (+ RURAL/URBAN at the end),
+# so a whole run can be pasted in one block with NO formulas left in Excel.
+SUMMARY_COLUMNS = [
+    "Date Processed", "State", "County", "FIPS",
+    "Overall Total Dataset City Assignment", "Overall Total Undetermined City Assignment",
+    "Overall Total Dataset Zip Assignment", "Overall Total Undetermined Zip Assignment",
+    "Overall Total Dataset Census Tract Assignment", "Overall Total Undetermined Census Tract Assignment",
+    "Total Records in County", "Total Subdivisions In County", "Total Clusters Made",
+    "Total Undetermined City", "Total City Assignments", "Total Undetermined Zip",
+    "Total Zip Assignments", "Total Blank Census", "Total Census Assignments",
+    "Added By", "Processing Times/Notes", "Run", "RURAL/URBAN",
+    "Total Blanks before run", "% of Total Blanks before run",
+    "Total Blanks after run", "% of Total Blanks after run",
+]
+
+
+def _pct(num, den):
+    """Percentage num/den*100, rounded to 2 dp; 0 when the denominator is 0
+    (replaces the Excel formulas, which would show #DIV/0! in that case)."""
+    return round(num / den * 100, 2) if den else 0
+
+
+def _summary_row(name, fips, records, subdivisions, clusters,
+                 undet_city, city_assign, undet_zip, zip_assign, blank_census, census_assign):
+    """Build one fully-computed Summary row matching SUMMARY_COLUMNS. Percentages,
+    run date, state, and RURAL/URBAN are all computed here so the tracking file
+    needs no formulas. Used by BOTH the streaming and memory paths (no drift)."""
+    blanks_before = undet_city + undet_zip + blank_census
+    filled = city_assign + zip_assign + census_assign
+    blanks_after = blanks_before - filled          # blanks the program couldn't resolve
+    return {
+        "Date Processed": time.strftime("%m/%d/%Y"),
+        "State": STATE_FIPS_ABBR.get(str(fips).strip()[:2], ""),
+        "County": name,
+        "FIPS": fips,
+        "Overall Total Dataset City Assignment": _pct(city_assign, records),
+        "Overall Total Undetermined City Assignment": _pct(city_assign, undet_city),
+        "Overall Total Dataset Zip Assignment": _pct(zip_assign, records),
+        "Overall Total Undetermined Zip Assignment": _pct(zip_assign, undet_zip),
+        "Overall Total Dataset Census Tract Assignment": _pct(census_assign, records),
+        "Overall Total Undetermined Census Tract Assignment": _pct(census_assign, blank_census),
+        "Total Records in County": records,
+        "Total Subdivisions In County": subdivisions,
+        "Total Clusters Made": clusters,
+        "Total Undetermined City": undet_city,
+        "Total City Assignments": city_assign,
+        "Total Undetermined Zip": undet_zip,
+        "Total Zip Assignments": zip_assign,
+        "Total Blank Census": blank_census,
+        "Total Census Assignments": census_assign,
+        "Added By": "",
+        "Processing Times/Notes": "",
+        "Run": "",
+        "RURAL/URBAN": "RURAL" if records <= RURAL_MAX_RECORDS else "URBAN",
+        "Total Blanks before run": blanks_before,
+        "% of Total Blanks before run": round(blanks_before / records, 2) if records else 0,
+        "Total Blanks after run": blanks_after,
+        "% of Total Blanks after run": round(blanks_after / records, 2) if records else 0,
+    }
+
 
 def _mem_mb():
     """Current process memory (RSS) in MB. psutil if available, else peak RSS."""
@@ -1012,29 +1089,21 @@ def run_pipeline(uploaded_files, reference_file, blank_cities_not_in_list,
                                                   _blank(g, ORIG_ZIP), _blank(g, ORIG_TRACT))
             city_fill, zip_fill, tract_fill = (_fill(g, RW_CITY),
                                                _fill(g, RW_ZIP), _fill(g, RW_TRACT))
-            summary_rows.append({
-                "County": name,
-                "FIPS": fips,
-                # true totals from the full frame
-                "Total Records in County": len(gfull),
-                # how many of those records carry a LegalSubDivision (non-unique)
-                "Total Subdivisions": int(gfull["LegalSubDivision"].count())
-                                      if "LegalSubDivision" in gfull.columns else 0,
-                "Total Clusters Made": int(g["Cluster_ID"].nunique())
-                                       if "Cluster_ID" in g.columns else 0,
-                "Total Blank Cities": int(city_blank.sum()),
-                "Total Cities Assignment": int((city_blank & city_fill).sum()),
-                "Total Blank Zips": int(zip_blank.sum()),
-                "Total Zips Assignment": int((zip_blank & zip_fill).sum()),
-                "Total Blank Census": int(tract_blank.sum()),
-                "Total Census Assignment": int((tract_blank & tract_fill).sum()),
-            })
-
-    summary_cols = ["County", "FIPS", "Total Records in County", "Total Subdivisions",
-                    "Total Clusters Made", "Total Blank Cities", "Total Cities Assignment",
-                    "Total Blank Zips", "Total Zips Assignment", "Total Blank Census",
-                    "Total Census Assignment"]
-    summary_df = pd.DataFrame(summary_rows, columns=summary_cols)
+            summary_rows.append(_summary_row(
+                name, fips,
+                records=len(gfull),
+                subdivisions=int(gfull["LegalSubDivision"].count())
+                             if "LegalSubDivision" in gfull.columns else 0,
+                clusters=int(g["Cluster_ID"].nunique()) if "Cluster_ID" in g.columns else 0,
+                undet_city=int(city_blank.sum()),
+                city_assign=int((city_blank & city_fill).sum()),
+                undet_zip=int(zip_blank.sum()),
+                zip_assign=int((zip_blank & zip_fill).sum()),
+                blank_census=int(tract_blank.sum()),
+                census_assign=int((tract_blank & tract_fill).sum()),
+            ))
+
+    summary_df = pd.DataFrame(summary_rows, columns=SUMMARY_COLUMNS)
     _step(log, f"Summary: {len(summary_df)} county row(s).")
 
     summary_path = out_dir / "Summary.csv"
@@ -1284,18 +1353,18 @@ def process_county(cdf, wl, fips_col, cluster_offset,
                                           _blank(g, "DDPPropertyZIPFinal"), _blank(g, "DDPRevisedCensusTract"))
     city_fill, zip_fill, tract_fill = (_fill(g, "Replace with Cities"),
                                        _fill(g, "Replace with Zip Codes"), _fill(g, "Replace with Census Tracts"))
-    summary_row = {
-        "County": name, "FIPS": fips,
-        "Total Records in County": len(cdf),
-        "Total Subdivisions": int(cdf["LegalSubDivision"].count()) if "LegalSubDivision" in cdf.columns else 0,
-        "Total Clusters Made": n_clusters,
-        "Total Blank Cities": int(city_blank.sum()),
-        "Total Cities Assignment": int((city_blank & city_fill).sum()),
-        "Total Blank Zips": int(zip_blank.sum()),
-        "Total Zips Assignment": int((zip_blank & zip_fill).sum()),
-        "Total Blank Census": int(tract_blank.sum()),
-        "Total Census Assignment": int((tract_blank & tract_fill).sum()),
-    }
+    summary_row = _summary_row(
+        name, fips,
+        records=len(cdf),
+        subdivisions=int(cdf["LegalSubDivision"].count()) if "LegalSubDivision" in cdf.columns else 0,
+        clusters=n_clusters,
+        undet_city=int(city_blank.sum()),
+        city_assign=int((city_blank & city_fill).sum()),
+        undet_zip=int(zip_blank.sum()),
+        zip_assign=int((zip_blank & zip_fill).sum()),
+        blank_census=int(tract_blank.sum()),
+        census_assign=int((tract_blank & tract_fill).sum()),
+    )
 
     return {
         "validated": cdf, "audit": audit, "cluster_df": cluster_df,
@@ -1482,11 +1551,7 @@ def run_streaming(uploaded_files, reference_file, blank_cities_not_in_list,
     outputs.extend(output_parts)
     outputs.extend(bad_files)   # malformed-row files for review
 
-    summary_cols = ["County", "FIPS", "Total Records in County", "Total Subdivisions",
-                    "Total Clusters Made", "Total Blank Cities", "Total Cities Assignment",
-                    "Total Blank Zips", "Total Zips Assignment", "Total Blank Census",
-                    "Total Census Assignment"]
-    summary_df = pd.DataFrame(summary_rows, columns=summary_cols)
+    summary_df = pd.DataFrame(summary_rows, columns=SUMMARY_COLUMNS)
     summary_path = out_dir / "Summary.csv"
     summary_df.to_csv(summary_path, index=False)
     outputs.append(str(summary_path))