Add interactive schedule_o notebook from Databricks

notebooks/NP03_schedule_I.ipynb

@@ -0,0 +1,635 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "939230d3-02ed-43f2-a10a-e983c2c23964",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "source": [
+    "#Funding organizations - 990 Filers\n",
+    "\n",
+    "Schedule I is completed by organizations who answer \"Yes\" on Form 990, Part IV, line 21 or 22.\n",
+    "\n",
+    "Question 21:  \n",
+    "\"Did the organization report more than $5,000 of grants or other assistance to any domestic organization or domestic government on Part IX, column (A), line 1? If “Yes,” complete Schedule I, Parts I and II\"\n",
+    "\n",
+    "Question 22:  \n",
+    "\"Did the organization report more than $5,000 of grants or other assistance to or for domestic individuals on Part IX, column (A), line 2? If “Yes,” complete Schedule I, Parts I and III\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "9dc61c58-ae98-4c6a-b8a2-cfd67b848b61",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from pyspark.sql import functions as F\n",
+    "from pyspark.sql.window import Window"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "d052b039-2f34-48ce-851e-416a15697695",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "scheduleigrantsp2 = spark.table(\"prod_curated.irs.scheduleipart2grants\")\n",
+    "form990cn120fields = spark.table(\"prod_curated.irs.990standardfields\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "71943bff-f143-4481-82b7-8dfc0c3aa965",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "source": [
+    "##Data checks"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "29735c68-9c07-49b7-9a37-816d82de393b",
+     "showTitle": true,
+     "tableResultSettingsMap": {},
+     "title": "Check for non-unique filer state records within a tax year"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# filer_states = (\n",
+    "#     form990cn120fields\n",
+    "#     .select(\n",
+    "#         'FILEREIN',\n",
+    "#         'TAXYEAR',\n",
+    "#         'FILERUSSTATE',\n",
+    "#     )\n",
+    "#     .distinct()\n",
+    "#     .groupBy('FILEREIN', 'TAXYEAR')\n",
+    "#     .agg(F.countDistinct('FILERUSSTATE').alias('state_count'))\n",
+    "#     .filter(F.col('state_count') > 1)\n",
+    "# )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "e8dea51f-9016-4a6d-b199-52d5509c6f32",
+     "showTitle": true,
+     "tableResultSettingsMap": {},
+     "title": "Check for non-unique filer city records within a tax year"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# filer_cities = (\n",
+    "#     form990cn120fields\n",
+    "#     .select(\n",
+    "#         'FILEREIN',\n",
+    "#         'TAXYEAR',\n",
+    "#         'FILERUSCITY',\n",
+    "#     )\n",
+    "#     .distinct()\n",
+    "#     .groupBy('FILEREIN', 'TAXYEAR')\n",
+    "#     .agg(F.countDistinct('FILERUSCITY').alias('state_count'))\n",
+    "#     .filter(F.col('state_count') > 1)\n",
+    "# )\n",
+    "\n",
+    "# display(filer_cities)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "d928d099-3b12-4c27-af21-bb85f83245d4",
+     "showTitle": true,
+     "tableResultSettingsMap": {},
+     "title": "Check recipient state counts"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "display(\n",
+    "    scheduleigrantsp2_cl\n",
+    "    .groupBy(\n",
+    "        'RECTABADDSTA'\n",
+    "    ).agg(\n",
+    "        F.count('*')\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "# display(\n",
+    "#     scheduleigrantsp2_cl\n",
+    "#     .filter(\n",
+    "#         F.col('RECTABADDSTA') == 'AA'\n",
+    "#     )\n",
+    "# )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "29713546-0dcb-4b5f-8717-0232b4f617d0",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "display(scheduleigrantsp2.filter(F.col('RECTABADDSTA').isNull()))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "9c2e202d-cb25-4753-8b44-0adfa0a97132",
+     "showTitle": true,
+     "tableResultSettingsMap": {},
+     "title": "Check overlap of null grant amounts versus null state codes"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# cross_section_counts = (\n",
+    "#     scheduleigrantsp2_cl\n",
+    "#     .select(\n",
+    "#         F.when(F.col('RECTABADDSTA').isNull(), 'NULL').otherwise('NON_NULL').alias('RECTABADDSTA_status'),\n",
+    "#         F.when(F.col('RETAAMOFCAGR').isNull(), 'NULL').otherwise('NON_NULL').alias('RETAAMOFCAGR_status')\n",
+    "#     )\n",
+    "#     .groupBy(\n",
+    "#         'RECTABADDSTA_status', 'RETAAMOFCAGR_status'\n",
+    "#     ).agg(\n",
+    "#         F.count('*').alias('count')\n",
+    "#     )\n",
+    "# )\n",
+    "\n",
+    "# display(cross_section_counts)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "da3ea6fa-5d11-437e-8705-824ec6ea5d5a",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "source": [
+    "##Schedule I Filers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "253b21d0-e9c6-4eea-b74a-942a96192e7f",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "filer_states = (\n",
+    "    form990cn120fields\n",
+    "    .select(\n",
+    "        'FILEREIN',\n",
+    "        'TAXYEAR',\n",
+    "        'FILERUSSTATE',\n",
+    "    ).distinct()\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "ae49acb7-fc82-4b25-8644-2f1798ac70aa",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "filer_cities = (\n",
+    "    form990cn120fields\n",
+    "    .select(\n",
+    "        'FILEREIN',\n",
+    "        'TAXYEAR',\n",
+    "        'FILERUSCITY',\n",
+    "    ).distinct()\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "952f0e50-716e-42cc-abb7-b88619c7fb86",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "scheduleigrantsp2_cl = (\n",
+    "    scheduleigrantsp2\n",
+    "    .select(\n",
+    "        'FILEREIN',\n",
+    "        'TAXYEAR',\n",
+    "        'RTEINORECIPI',\n",
+    "        'RTRNBBNLINE11',\n",
+    "        'RECTABADDCIT',\n",
+    "        'RECTABADDSTA',\n",
+    "        'RETAAMOFCAGR',\n",
+    "    )\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "7f6df2c9-25a9-4eb0-9057-043e77d9b11d",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "filer_window = Window.partitionBy('FILEREIN', 'TAXYEAR')\n",
+    "state_rank_window = Window.partitionBy('FILEREIN', 'TAXYEAR').orderBy(F.desc('total_grant_value'))\n",
+    "\n",
+    "grants_per_state = (\n",
+    "    scheduleigrantsp2_cl\n",
+    "    .filter(\n",
+    "        F.col('RECTABADDSTA').isNotNull()\n",
+    "    )\n",
+    "    .groupBy(\n",
+    "        'FILEREIN', 'TAXYEAR', 'RECTABADDSTA'\n",
+    "    ).agg(\n",
+    "        F.sum('RETAAMOFCAGR').alias('total_grant_value'),\n",
+    "        F.count('*').alias('total_grant_count'),\n",
+    "    ).withColumn(\n",
+    "        'total',\n",
+    "        F.sum('total_grant_value').over(filer_window)\n",
+    "    ).filter(\n",
+    "        F.col('total') > 0\n",
+    "    ).withColumn(\n",
+    "        'proportion',\n",
+    "        F.col('total_grant_value') / F.col('total')\n",
+    "    ).withColumn(\n",
+    "        'total_states',\n",
+    "        F.count('*').over(filer_window)\n",
+    "    ).withColumn(\n",
+    "        'rank',\n",
+    "        F.rank().over(state_rank_window)  # Rank states within each filer-year based on grant value\n",
+    "    ).groupBy(\n",
+    "        'FILEREIN', 'TAXYEAR'\n",
+    "    ).agg(\n",
+    "        F.sum('total_grant_value').alias('total_grant_value'),\n",
+    "        F.sum('total_grant_count').alias('total_grant_count'),\n",
+    "        F.first('total_states').alias('total_recipient_states'),\n",
+    "        F.max('proportion').alias('max_recipient_state_percentage'),\n",
+    "        F.collect_set('RECTABADDSTA').alias('distinct_recipient_states'),  # Collect unique states into a list\n",
+    "        F.first(F.when(F.col('rank') == 1, F.col('RECTABADDSTA')), ignorenulls=True).alias('top_recipient_state'),  # Get state with highest grant value\n",
+    "    ).join(\n",
+    "        filer_states,\n",
+    "        on=['FILEREIN', 'TAXYEAR'],\n",
+    "        how='left'\n",
+    "    ).withColumn(\n",
+    "        'foreign_percentage',   # Schedule I is for domestic grants so assumed 0 foreign\n",
+    "        F.lit(0)                # Added column for combining datasets later on\n",
+    "    ).select(\n",
+    "        'FILEREIN',\n",
+    "        'TAXYEAR',\n",
+    "        'FILERUSSTATE',\n",
+    "        'total_grant_value',\n",
+    "        'total_grant_count',\n",
+    "        'total_recipient_states',\n",
+    "        'foreign_percentage',\n",
+    "        'max_recipient_state_percentage',\n",
+    "        'distinct_recipient_states',\n",
+    "        'top_recipient_state',\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "display(grants_per_state)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "9772faeb-640c-4bdb-9a99-8b0e41f3bbda",
+     "showTitle": true,
+     "tableResultSettingsMap": {},
+     "title": "Aggregation at city level could be used to identify funders with local activity"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "filer_window = Window.partitionBy('FILEREIN', 'TAXYEAR')\n",
+    "city_rank_window = Window.partitionBy('FILEREIN', 'TAXYEAR').orderBy(F.desc('total_grant_value'))\n",
+    "\n",
+    "grants_per_city = (\n",
+    "    scheduleigrantsp2_cl\n",
+    "    .filter(\n",
+    "        F.col('RECTABADDCIT').isNotNull()\n",
+    "    )\n",
+    "    .groupBy(\n",
+    "        'FILEREIN', 'TAXYEAR', 'RECTABADDCIT'\n",
+    "    ).agg(\n",
+    "        F.sum('RETAAMOFCAGR').alias('total_grant_value'),\n",
+    "        F.count('*').alias('total_grant_count'),\n",
+    "    ).withColumn(\n",
+    "        'total',\n",
+    "        F.sum('total_grant_value').over(filer_window)\n",
+    "    ).filter(\n",
+    "        F.col('total') > 0\n",
+    "    ).withColumn(\n",
+    "        'proportion',\n",
+    "        F.col('total_grant_value') / F.col('total')\n",
+    "    ).withColumn(\n",
+    "        'total_cities',\n",
+    "        F.count('*').over(filer_window)\n",
+    "    ).withColumn(\n",
+    "        'rank',\n",
+    "        F.rank().over(city_rank_window)  # Rank cities within each filer-year based on grant value\n",
+    "    ).groupBy(\n",
+    "        'FILEREIN', 'TAXYEAR'\n",
+    "    ).agg(\n",
+    "        F.sum('total_grant_value').alias('total_grant_value'),\n",
+    "        F.sum('total_grant_count').alias('total_grant_count'),\n",
+    "        F.first('total_cities').alias('total_recipient_cities'),\n",
+    "        F.max('proportion').alias('max_recipient_city_percentage'),\n",
+    "        F.collect_set('RECTABADDCIT').alias('distinct_recipient_cities'),  # Collect unique cities into a list\n",
+    "        F.first(F.when(F.col('rank') == 1, F.col('RECTABADDCIT')), ignorenulls=True).alias('top_recipient_city'),  # Get city with highest grant value\n",
+    "    ).join(\n",
+    "        filer_cities,\n",
+    "        on=['FILEREIN', 'TAXYEAR'],\n",
+    "        how='left'\n",
+    "    ).select(\n",
+    "        'FILEREIN',\n",
+    "        'TAXYEAR',\n",
+    "        'FILERUSCITY',\n",
+    "        'total_grant_value',\n",
+    "        'total_grant_count',\n",
+    "        'total_recipient_cities',\n",
+    "        'max_recipient_city_percentage',\n",
+    "        'distinct_recipient_cities',\n",
+    "        'top_recipient_city',\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "display(grants_per_city)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "74a921cc-2325-4a27-88a6-5cfa28f52ac0",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "grants_per_state = (\n",
+    "    grants_per_state.withColumn('source', F.lit('990 (domestic grants, schedule I)'))\n",
+    ")\n",
+    "grants_per_state.write.mode('overwrite').saveAsTable('sandbox_edward.nonprofit_mapping.grants_per_state_990_filers')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "78af447a-fd18-4f6d-8283-a431e2d82435",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "source": [
+    "##International activity\n",
+    "\n",
+    "Would ideally use data in Schedule F but this isn't in Databricks yet.  \n",
+    "\n",
+    "Form 990 questions 14a, 14b, 15, and 16 could be used to identify foregin activity but this information seems to be missing from prod_curated.irs.990cn120fields:  \n",
+    "F9_04_PC_FOREIGOFFICE (14a)  \n",
+    "F9_04_PC_FOREIGACTIVI (14b)  \n",
+    "F9_04_PC_MOTHKTTOORIN (15)  \n",
+    "F9_04_PC_MOTHKTTOORIN (15)  \n",
+    "F9_04_PC_MOTHKTTOININ (16)  \n",
+    "F9_04_PC_MOTHKTTOINND (16)  \n",
+    "\n",
+    "prod_curated.irs.990cn120fields does contain the field F9_09_PC_FOREGRANTOTA from part 9 of the form which totals the amounts given in foreign grants"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "30d7ddc0-df13-4683-98f5-7b24b918e01c",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "foreign_activity =(\n",
+    "    form990cn120fields\n",
+    "    .select(\n",
+    "        'FILEREIN',\n",
+    "        'TAXYEAR',\n",
+    "        'FOREGRANTOTA',\n",
+    "    ).filter(\n",
+    "        F.col('FOREGRANTOTA') > 0\n",
+    "    )\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "582caba3-4d4d-4e64-b222-eb7af12ce6bd",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "display(foreign_activity)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "00451a40-f571-4bbc-820b-d6fd31b3537a",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "computePreferences": null,
+   "dashboards": [],
+   "environmentMetadata": {
+    "base_environment": "",
+    "environment_version": "2"
+   },
+   "inputWidgetPreferences": null,
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 4
+   },
+   "notebookName": "(Clone) NP03_schedule_I",
+   "widgets": {}
+  },
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}