{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "62d4799f-4935-4a2d-8f0a-5f6383b22cf7",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": [
"df1 = spark.read.table(\"prod_curated.irs.990cn120fields\")\n",
"df2 = spark.read.table(\"prod_curated.irs.990standardfields\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "d5410f3d-7463-43f5-8bfb-528d36e80b42",
"showTitle": false,
"tableResultSettingsMap": {
"0": {
"dataGridStateBlob": "{\"version\":1,\"tableState\":{\"columnPinning\":{\"left\":[\"#row_number#\"],\"right\":[]},\"columnSizing\":{\"column\":116},\"columnVisibility\":{}},\"settings\":{\"columns\":{}},\"syncTimestamp\":1758734440525}",
"filterBlob": null,
"queryPlanFiltersBlob": null,
"tableResultIndex": 0
}
},
"title": ""
}
},
"outputs": [],
"source": [
"from pyspark.sql import SparkSession\n",
"import pandas as pd\n",
"\n",
"# Extract (col, dtype) as dicts\n",
"df1_schema = {f.name: f.dataType.simpleString() for f in df1.schema.fields}\n",
"df2_schema = {f.name: f.dataType.simpleString() for f in df2.schema.fields}\n",
"\n",
"# Union of all column names\n",
"all_cols = set(df1_schema.keys()).union(df2_schema.keys())\n",
"\n",
"# Build comparison rows\n",
"rows = []\n",
"for col in sorted(all_cols):\n",
" in_df1 = col in df1_schema\n",
" in_df2 = col in df2_schema\n",
" \n",
" if in_df1 and in_df2:\n",
" flag = \"both\"\n",
" elif in_df1:\n",
" flag = \"old\"\n",
" else:\n",
" flag = \"new\"\n",
" \n",
" rows.append({\n",
" \"column\": col,\n",
" \"in_df\": flag,\n",
" \"dtype_old\": df1_schema.get(col),\n",
" \"dtype_new\": df2_schema.get(col)\n",
" })\n",
"\n",
"# Convert to pandas for inspection\n",
"comparison_df = pd.DataFrame(rows)\n",
"\n",
"# If you prefer it as a Spark DataFrame:\n",
"spark = SparkSession.builder.getOrCreate()\n",
"spark_comparison_df = spark.createDataFrame(comparison_df)\n",
"\n",
"display(comparison_df)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "2634e810-1046-456f-a00e-34db0ca198a2",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": [
"from pyspark.sql import functions as F\n",
"from pyspark.sql.window import Window\n",
"\n",
"from pyspark.ml.feature import VectorAssembler, StandardScaler\n",
"from pyspark.ml.clustering import KMeans\n",
"\n",
"import plotly.express as px"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "cc50ff8a-e01c-417d-b926-fecac95265d0",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": [
"grants_per_state_990 = spark.read.table('sandbox_edward.nonprofit_mapping.grants_per_state_990_filers')\n",
"grants_per_state_990pf = spark.read.table('sandbox_edward.nonprofit_mapping.grants_per_state_990pf_filers')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "79330be0-c72e-4670-b6bd-b95665af55c8",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "check for EINs in both 990 and 990pf"
}
},
"outputs": [],
"source": [
"dual_filers = (\n",
" grants_per_state_990.select(\n",
" 'FILEREIN', \n",
" 'TAXYEAR'\n",
" )\n",
" .join(\n",
" grants_per_state_990pf.select('FILEREIN', 'TAXYEAR'), \n",
" on=['FILEREIN', 'TAXYEAR'],\n",
" how='inner'\n",
" )\n",
")\n",
"\n",
"display(dual_filers)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "3cd453d0-0bac-42d7-b9d3-51a30be32e6b",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": [
"display(grants_per_state_990.filter(F.col('FILEREIN')=='85-0462315'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "7232db59-693a-43d9-826b-9e6c2a271626",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": [
"display(grants_per_state_990pf.filter(F.col('FILEREIN')=='85-0462315'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "64686496-b51f-4aad-ad34-f88e2b69cf61",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "drop dual filers"
}
},
"outputs": [],
"source": [
"grants_per_state_990 = grants_per_state_990.join(\n",
" dual_filers.select(F.col('FILEREIN'), F.col('TAXYEAR')),\n",
" on=['FILEREIN', 'TAXYEAR'],\n",
" how='left_anti'\n",
")\n",
"\n",
"grants_per_state_990pf = grants_per_state_990pf.join(\n",
" dual_filers.select(F.col('FILEREIN'), F.col('TAXYEAR')),\n",
" on=['FILEREIN', 'TAXYEAR'],\n",
" how='left_anti'\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "f0f23c24-a091-49d6-bc7b-64596c89ed0a",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "combine 990 & 990pf orgs into one df"
}
},
"outputs": [],
"source": [
"grants_per_state = grants_per_state_990.union(grants_per_state_990pf).orderBy('FILEREIN', 'TAXYEAR')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "ac11e27e-18a2-41e0-b8a2-2f6889990a02",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": [
"display(grants_per_state)"
]
},
{
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "6b8aee01-6623-4556-a717-9f58d8af4b6e",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"source": [
"##KMeans Clustering"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "013375d3-74b9-48a2-9db0-fe70b09c47f5",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "feature engineering"
}
},
"outputs": [],
"source": [
"# Normalize/scale features\n",
"feature_cols = [\"foreign_percentage\", \"max_recipient_state_percentage\", \"total_recipient_states\"]\n",
"assembler = VectorAssembler(inputCols=feature_cols, outputCol=\"features_unscaled\")\n",
"df_features = assembler.transform(grants_per_state)\n",
"\n",
"scaler = StandardScaler(inputCol=\"features_unscaled\", outputCol=\"features\", withStd=True, withMean=True)\n",
"df_scaled = scaler.fit(df_features).transform(df_features)\n",
"\n",
"# Create a composite score - optional, may not add value\n",
"# max_states = grants_per_state.select(F.max('total_recipient_states')).collect()[0][0]\n",
"# grants_per_state = grants_per_state.withColumn(\n",
"# \"composite_score\",\n",
"# 0.5 * (1 - F.col(\"max_recipient_state_percentage\")/100) + \n",
"# 0.3 * (F.col(\"total_recipient_states\")/max_states) + \n",
"# 0.2 * (F.col(\"foreign_percentage\")/100)\n",
"# )\n",
"# feature_cols = [\"foreign_percentage\", \"max_recipient_state_percentage\", \"total_recipient_states\", \"composite_score\"]\n",
"# assembler = VectorAssembler(inputCols=feature_cols, outputCol=\"features_unscaled\")\n",
"# df_features = assembler.transform(grants_per_state)\n",
"\n",
"# scaler = StandardScaler(inputCol=\"features_unscaled\", outputCol=\"features\", withStd=True, withMean=True)\n",
"# df_scaled = scaler.fit(df_features).transform(df_features)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "9bedd689-bb2c-4beb-9f7b-1756ba0c99c5",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "clustering"
}
},
"outputs": [],
"source": [
"# Clustering on all the scaled features\n",
"kmeans = KMeans(featuresCol=\"features\", predictionCol=\"cluster\", k=3, seed=42)\n",
"model = kmeans.fit(df_scaled)\n",
"\n",
"# Assign clusters\n",
"df_clustered = model.transform(df_scaled)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "4f47c4f6-2143-41a7-b921-55cc3405be3a",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": [
"display(df_clustered)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "5cce9fe9-0c60-4c5d-971e-1460e813a0fc",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": [
"# df_clustered.write.mode('overwrite').saveAsTable('sandbox_edward.nonprofit_mapping.funding_orgs_local_vs_national_kmeans_with_composite_feature')\n",
"df_clustered.write.mode('overwrite').saveAsTable('sandbox_edward.nonprofit_mapping.funding_orgs_local_vs_national_kmeans_without_composite_feature')"
]
},
{
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "86298a0c-3526-4749-84d8-33c4119da0d8",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"source": [
"##Cluster Summary - With Composite Feature"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "702ac066-0d69-47e9-9411-f080d3a541ea",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": [
"df_clustered = spark.read.table('sandbox_edward.nonprofit_mapping.funding_orgs_local_vs_national_kmeans_with_composite_feature')"
]
},
{
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "3926601d-c18f-4fed-a87e-06b762d61c6e",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"source": [
"Cluster 0 = local/regional
\n",
"Cluster 1 = international
\n",
"Cluster 2 = national"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "5ae9099f-f596-4565-954a-40035f6eb880",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "summarize clusters by original features"
}
},
"outputs": [],
"source": [
"summary = (\n",
" df_clustered\n",
" .groupBy(\"cluster\")\n",
" .agg(\n",
" F.count(\"*\").alias(\"count\"),\n",
" F.avg(\"foreign_percentage\").alias(\"avg_foreign_percentage\"),\n",
" F.median(\"foreign_percentage\").alias(\"median_foreign_percentage\"),\n",
" F.min(\"foreign_percentage\").alias(\"min_foreign_percentage\"),\n",
" F.max(\"foreign_percentage\").alias(\"max_foreign_percentage\"),\n",
" F.avg(\"max_recipient_state_percentage\").alias(\"avg_max_state_pct\"),\n",
" F.median(\"max_recipient_state_percentage\").alias(\"median_max_state_pct\"),\n",
" F.min(\"max_recipient_state_percentage\").alias(\"min_max_state_pct\"),\n",
" F.max(\"max_recipient_state_percentage\").alias(\"max_max_state_pct\"),\n",
" F.avg(\"total_recipient_states\").alias(\"avg_distinct_states\"),\n",
" F.median(\"total_recipient_states\").alias(\"median_distinct_states\"),\n",
" F.min(\"total_recipient_states\").alias(\"min_distinct_states\"),\n",
" F.max(\"total_recipient_states\").alias(\"max_distinct_states\"),\n",
" )\n",
" .orderBy(\"cluster\")\n",
")\n",
"\n",
"display(summary)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "def2f16f-982e-448e-9aa8-782aa01c2193",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "create distribution plots for each cluster (feature: foreign percentage)"
}
},
"outputs": [],
"source": [
"pdf_clustered = df_clustered.toPandas()\n",
"\n",
"fig_foreign = px.box(\n",
" pdf_clustered,\n",
" x=\"cluster\",\n",
" y=\"foreign_percentage\",\n",
" title=\"Foreign Percentage by Cluster\",\n",
" labels={\"foreign_percentage\": \"Foreign Percentage\", \"cluster\": \"Cluster\"}\n",
")\n",
"fig_foreign.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "01020c54-f610-42e2-b43c-f2643f98a576",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "create distribution plots for each cluster (feature: max recipient state percentage)"
}
},
"outputs": [],
"source": [
"fig_max_recipient = px.box(\n",
" pdf_clustered,\n",
" x=\"cluster\",\n",
" y=\"max_recipient_state_percentage\",\n",
" title=\"Max Recipient State Percentage by Cluster\",\n",
" labels={\"max_recipient_state_percentage\": \"Max Recipient State Percentage\", \"cluster\": \"Cluster\"}\n",
")\n",
"fig_max_recipient.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "170d71cf-cb3f-44d5-bf90-cc9746a3c1d3",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "create distribution plots for each cluster (feature: number of states)"
}
},
"outputs": [],
"source": [
"fig_total_states = px.box(\n",
" pdf_clustered,\n",
" x=\"cluster\",\n",
" y=\"total_recipient_states\",\n",
" title=\"Total Recipient States by Cluster\",\n",
" labels={\"total_recipient_states\": \"Total Recipient States\", \"cluster\": \"Cluster\"}\n",
")\n",
"fig_total_states.show()"
]
},
{
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "12dc14fa-0066-4fe5-8a99-d9c6d05860aa",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"source": [
"##Cluster Summary - Without Composite Feature"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "1545cd83-3e0a-43f9-9719-14d0f12f5dcb",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": [
"df_clustered = spark.read.table('sandbox_edward.nonprofit_mapping.funding_orgs_local_vs_national_kmeans_without_composite_feature')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "f4a8fe26-50d1-4d55-bd18-0313a1d55136",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": [
"display(df_clustered)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "6b2a98fa-d5ee-4c8f-8aad-208a83b2d145",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": [
"from pyspark.sql import functions as F\n",
"\n",
"df_2023 = (\n",
" df_clustered\n",
" .filter(F.col(\"TAXYEAR\") == 2023)\n",
" .withColumn(\"locality\", F.when(F.col(\"cluster\")==0, \"local/regional\").otherwise(F.when(F.col(\"cluster\")==1, \"international\").otherwise(F.when(F.col(\"cluster\")==2, \"national\").otherwise(None))))\n",
" .select(\n",
" \"FILEREIN\",\n",
" \"TAXYEAR\",\n",
" \"FILERUSSTATE\",\n",
" F.col(\"total_grant_value\").alias(\"value_of_grants\"),\n",
" F.col(\"total_grant_count\").alias(\"number_of_grants\"),\n",
" F.col(\"total_recipient_states\").alias(\"number_of_recipient_states\"),\n",
" F.col(\"foreign_percentage\").alias(\"pct_grant_value_foreign\"),\n",
" F.col(\"max_recipient_state_percentage\").alias(\"pct_grant_value_top_state\"),\n",
" F.col(\"top_recipient_state\").alias(\"top_state\"),\n",
" F.col(\"distinct_recipient_states\").alias(\"recipient_states\"),\n",
" \"locality\",\n",
" \"source\",\n",
" )\n",
")\n",
"display(df_2023)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "2483cf1c-4bed-47f0-91c0-0364e0f0d5da",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": [
"df_2023.write.mode(\"overwrite\").saveAsTable(\"sandbox_edward.nonprofit_mapping.locality_by_granting_activity_segmentation_funding_orgs_taxyear2023\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "0781ad05-93a0-46fe-9357-48fea2039b81",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"source": [
"Cluster 0 = local/regional
\n",
"Cluster 1 = international
\n",
"Cluster 2 = national"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "840cdfdb-dbad-4264-b0fc-85bb060ac2aa",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "summarize clusters by original features"
}
},
"outputs": [],
"source": [
"summary = (\n",
" df_clustered\n",
" .groupBy(\"cluster\")\n",
" .agg(\n",
" F.count(\"*\").alias(\"count\"),\n",
" F.avg(\"foreign_percentage\").alias(\"avg_foreign_percentage\"),\n",
" F.median(\"foreign_percentage\").alias(\"median_foreign_percentage\"),\n",
" F.min(\"foreign_percentage\").alias(\"min_foreign_percentage\"),\n",
" F.max(\"foreign_percentage\").alias(\"max_foreign_percentage\"),\n",
" F.avg(\"max_recipient_state_percentage\").alias(\"avg_max_state_pct\"),\n",
" F.median(\"max_recipient_state_percentage\").alias(\"median_max_state_pct\"),\n",
" F.min(\"max_recipient_state_percentage\").alias(\"min_max_state_pct\"),\n",
" F.max(\"max_recipient_state_percentage\").alias(\"max_max_state_pct\"),\n",
" F.avg(\"total_recipient_states\").alias(\"avg_distinct_states\"),\n",
" F.median(\"total_recipient_states\").alias(\"median_distinct_states\"),\n",
" F.min(\"total_recipient_states\").alias(\"min_distinct_states\"),\n",
" F.max(\"total_recipient_states\").alias(\"max_distinct_states\"),\n",
" )\n",
" .orderBy(\"cluster\")\n",
")\n",
"\n",
"display(summary)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "3e9a4fb8-11e3-4734-8d8e-943e03e3b738",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "create distribution plots for each cluster (feature: foreign percentage)"
}
},
"outputs": [],
"source": [
"pdf_clustered = df_clustered.toPandas()\n",
"\n",
"fig_foreign = px.box(\n",
" pdf_clustered,\n",
" x=\"cluster\",\n",
" y=\"foreign_percentage\",\n",
" title=\"Foreign Percentage by Cluster\",\n",
" labels={\"foreign_percentage\": \"Foreign Percentage\", \"cluster\": \"Cluster\"}\n",
")\n",
"fig_foreign.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "42bc6e79-cc9f-4745-9751-03c9223a3642",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "create distribution plots for each cluster (feature: max recipient state percentage)"
}
},
"outputs": [],
"source": [
"fig_max_recipient = px.box(\n",
" pdf_clustered,\n",
" x=\"cluster\",\n",
" y=\"max_recipient_state_percentage\",\n",
" title=\"Max Recipient State Percentage by Cluster\",\n",
" labels={\"max_recipient_state_percentage\": \"Max Recipient State Percentage\", \"cluster\": \"Cluster\"}\n",
")\n",
"fig_max_recipient.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "b05a45f8-4d81-4ac2-9cfb-b73e18d2051f",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "create distribution plots for each cluster (feature: number of states)"
}
},
"outputs": [],
"source": [
"fig_total_states = px.box(\n",
" pdf_clustered,\n",
" x=\"cluster\",\n",
" y=\"total_recipient_states\",\n",
" title=\"Total Recipient States by Cluster\",\n",
" labels={\"total_recipient_states\": \"Total Recipient States\", \"cluster\": \"Cluster\"}\n",
")\n",
"fig_total_states.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "3cf19964-c147-4cf4-b7b9-1f31a2e6a256",
"showTitle": false,
"tableResultSettingsMap": {},
"title": ""
}
},
"outputs": [],
"source": []
}
],
"metadata": {
"application/vnd.databricks.v1+notebook": {
"computePreferences": {
"hardware": {
"accelerator": null,
"gpuPoolId": null,
"memory": null
}
},
"dashboards": [],
"environmentMetadata": {
"base_environment": "",
"environment_version": "2"
},
"inputWidgetPreferences": null,
"language": "python",
"notebookMetadata": {
"pythonIndentUnit": 4
},
"notebookName": "(Clone) NP04_classification",
"widgets": {}
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}