Add interactive schedule_o notebook from Databricks

notebooks/NP02_990PF-grants-summary.ipynb

@@ -0,0 +1,344 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "eb152d62-ceb8-4b46-80d9-c77ceaca41a4",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from pyspark.sql import functions as F\n",
+    "from pyspark.sql.window import Window"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "82b0ab9b-6936-4513-8f67-68ecfd0b16f6",
+     "showTitle": true,
+     "tableResultSettingsMap": {},
+     "title": "Load table built in NP01_cleaning-steps-990PF-grants"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "grants3a_cleaned = spark.table(\"sandbox_edward.nonprofit_mapping.grants3a_cleaned\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "3f193ace-ecf5-4f55-b2a5-82b94eaf1a0e",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "#pfpart6astatesregistered = spark.table(\"prod_curated.irs.990pfpart6astatesregistered\")\n",
+    "pfstandardfields = spark.table(\"prod_curated.irs.990pfstandardfields\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "ca110791-727b-4c86-9194-46f27e976fcd",
+     "showTitle": true,
+     "tableResultSettingsMap": {},
+     "title": "Check for duplicate state registrations by filer / tax year"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "display(\n",
+    "    pfstandardfields\n",
+    "    .groupBy(\n",
+    "        'FILEREIN', 'TAXYEAR'\n",
+    "    ).agg(\n",
+    "        F.countDistinct('FILERUSSTATE').alias('filer_total_registered_states')\n",
+    "    ).orderBy(\n",
+    "        'filer_total_registered_states',\n",
+    "        ascending=False\n",
+    "    )\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "f20fa65c-77f1-4a40-9162-8ca929405d5d",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "filer_states = (\n",
+    "    pfstandardfields\n",
+    "    .select(\n",
+    "        'FILEREIN',\n",
+    "        'TAXYEAR',\n",
+    "        'FILERUSSTATE',\n",
+    "    ).distinct()\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "33e98457-6688-4349-8eff-d65aed0fef95",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "filer_window = Window.partitionBy('FILEREIN', 'TAXYEAR')\n",
+    "state_rank_window = Window.partitionBy('FILEREIN', 'TAXYEAR').orderBy(F.desc('proportion'))\n",
+    "\n",
+    "grants_per_state = (\n",
+    "    grants3a_cleaned\n",
+    "    # .filter(\n",
+    "    #     F.col('valid_state_code').isNotNull()\n",
+    "    # )\n",
+    "    .groupBy(\n",
+    "        'FILEREIN', 'TAXYEAR', 'valid_state_code',\n",
+    "    ).agg(\n",
+    "        F.sum('SIGOCPYAMOUN').alias('total_grant_value'),\n",
+    "        F.count('*').alias('total_grant_count'),\n",
+    "    ).withColumn(\n",
+    "        'total',\n",
+    "        F.sum('total_grant_value').over(filer_window)\n",
+    "    ).filter(\n",
+    "        F.col('total') > 0\n",
+    "    ).withColumn(\n",
+    "        'proportion',\n",
+    "        F.when(F.col('valid_state_code') != 'INTL', F.col('total_grant_value') / F.col('total')).otherwise(0)   # Exclude international grants - calculate separately\n",
+    "    ).withColumn(\n",
+    "        'total_states',\n",
+    "        F.sum(F.when(F.col(\"valid_state_code\") != \"INTL\", 1).otherwise(0)).over(filer_window)   # Count number of states, excluding international grants\n",
+    "    ).withColumn(\n",
+    "        'rank',\n",
+    "        F.rank().over(state_rank_window)  # Rank states within each filer-year based on grant value - rank by \"proportion\" to avoid INTL as the 1st state\n",
+    "    ).withColumn(\n",
+    "        'pct_foreign', \n",
+    "        F.when(F.col('valid_state_code') == 'INTL', F.col('total_grant_value') / F.col('total')).otherwise(0)   # Calculate percent of foreign grants\n",
+    "    ).groupBy(\n",
+    "        'FILEREIN', 'TAXYEAR',\n",
+    "    ).agg(\n",
+    "        F.sum('total_grant_value').alias('total_grant_value'),  # Includes foreign grants\n",
+    "        F.sum('total_grant_count').alias('total_grant_count'),  # Includes foreign grants\n",
+    "        F.first('total_states').alias('total_recipient_states'),\n",
+    "        F.max('pct_foreign').alias('foreign_percentage'),\n",
+    "        F.max('proportion').alias('max_recipient_state_percentage'),\n",
+    "        F.collect_set(F.when(F.col(\"valid_state_code\") != \"INTL\", F.col(\"valid_state_code\"))).alias('distinct_recipient_states'),  # Collect unique states into a list\n",
+    "        F.first(F.when(F.col('rank') == 1, F.col('valid_state_code')), ignorenulls=True).alias('top_recipient_state'),  # Get state with highest grant value (includes INTL)\n",
+    "    ).join(\n",
+    "        filer_states,\n",
+    "        on=['FILEREIN', 'TAXYEAR'],\n",
+    "        how='left'\n",
+    "    ).select(\n",
+    "        'FILEREIN',\n",
+    "        'TAXYEAR',\n",
+    "        'FILERUSSTATE',\n",
+    "        'total_grant_value',\n",
+    "        'total_grant_count',\n",
+    "        'total_recipient_states',\n",
+    "        'foreign_percentage',\n",
+    "        'max_recipient_state_percentage',\n",
+    "        'distinct_recipient_states',\n",
+    "        'top_recipient_state',\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "display(grants_per_state)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "793c9ddc-3d69-40f0-a228-c1f77dc80ab9",
+     "showTitle": true,
+     "tableResultSettingsMap": {},
+     "title": "Note: some FILEREINs don't have a record in pfstandardfields"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "display(pfstandardfields.filter(F.col('FILEREIN') == '01-0277832'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "4cc7ccb8-faa6-4321-926e-10f73df0b909",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "foreign_activity =(\n",
+    "    grants3a_cleaned\n",
+    "    .select(\n",
+    "        'FILEREIN',\n",
+    "        'TAXYEAR',\n",
+    "        # 'FOREGRANTOTA',\n",
+    "        'country_from_state_lookup',\n",
+    "    ).filter(\n",
+    "        F.col('country_from_state_lookup') == 'Assumed international'\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "display(foreign_activity)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "abe8358b-eac8-4bd2-8c55-b332eaec1988",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "display(grants3a_cleaned.filter(F.col('FILEREIN')=='01-0351827')) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "aa091446-359a-4d39-8346-777f9b4b2610",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "grants_per_state = (\n",
+    "    grants_per_state.withColumn('source', F.lit('990pf (grants, part XIV-3a)'))\n",
+    ")\n",
+    "grants_per_state.write.mode('overwrite').saveAsTable('sandbox_edward.nonprofit_mapping.grants_per_state_990pf_filers')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "b81634c6-e989-420f-8206-f358f0eed425",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "computePreferences": null,
+   "dashboards": [],
+   "environmentMetadata": {
+    "base_environment": "",
+    "environment_version": "2"
+   },
+   "inputWidgetPreferences": null,
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 4
+   },
+   "notebookName": "NP02_990PF-grants-summary",
+   "widgets": {}
+  },
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}