datamatters24
/

research-document-archive

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# 23 - Network Visualization\n",
+    "\n",
+    "Interactive notebook for visualizing entity co-occurrence networks.\n",
+    "\n",
+    "Loads the network JSON exported by `22_network_analysis` and renders an interactive\n",
+    "Plotly network graph with community coloring, centrality-based node sizing,\n",
+    "and edge thickness proportional to co-occurrence weight."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "sys.path.insert(0, '/opt/epstein_env/research')\n",
+    "\n",
+    "import json\n",
+    "import numpy as np\n",
+    "import networkx as nx\n",
+    "import plotly.graph_objects as go\n",
+    "from pathlib import Path\n",
+    "\n",
+    "from research_lib.config import WEB_ASSETS_DIR\n",
+    "from research_lib.plotting import set_style, save_fig, COLLECTION_COLORS"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Select which network to load\n",
+    "# Change this to visualize different collections\n",
+    "network_file = 'network_all.json'\n",
+    "\n",
+    "network_path = WEB_ASSETS_DIR / network_file\n",
+    "print(f'Loading network from: {network_path}')\n",
+    "\n",
+    "with open(network_path, 'r') as f:\n",
+    "    network_data = json.load(f)\n",
+    "\n",
+    "print(f'Nodes: {len(network_data[\"nodes\"])}')\n",
+    "print(f'Links: {len(network_data[\"links\"])}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Rebuild NetworkX graph from JSON for layout computation\n",
+    "G = nx.Graph()\n",
+    "\n",
+    "for node in network_data['nodes']:\n",
+    "    G.add_node(\n",
+    "        node['id'],\n",
+    "        label=node.get('label', node['id']),\n",
+    "        type=node.get('type', 'unknown'),\n",
+    "        size=node.get('size', 1),\n",
+    "        community=node.get('community', 0),\n",
+    "        centrality=node.get('centrality', 0),\n",
+    "    )\n",
+    "\n",
+    "for link in network_data['links']:\n",
+    "    G.add_edge(link['source'], link['target'], weight=link.get('weight', 1))\n",
+    "\n",
+    "print(f'Graph reconstructed: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Compute layout using spring layout (Fruchterman-Reingold)\n",
+    "print('Computing layout...')\n",
+    "pos = nx.spring_layout(G, k=1.5/np.sqrt(G.number_of_nodes()), iterations=50, seed=42, weight='weight')\n",
+    "print('Layout computed.')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Community color mapping\n",
+    "community_ids = sorted(set(G.nodes[n]['community'] for n in G.nodes))\n",
+    "# Generate distinct colors for communities\n",
+    "import plotly.express as px\n",
+    "color_palette = px.colors.qualitative.Set3 + px.colors.qualitative.Pastel + px.colors.qualitative.Bold\n",
+    "community_colors = {cid: color_palette[i % len(color_palette)] for i, cid in enumerate(community_ids)}\n",
+    "\n",
+    "print(f'Communities: {len(community_ids)}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Build edge traces\n",
+    "edge_traces = []\n",
+    "weights = [G.edges[u, v]['weight'] for u, v in G.edges]\n",
+    "max_weight = max(weights) if weights else 1\n",
+    "\n",
+    "edge_x = []\n",
+    "edge_y = []\n",
+    "for u, v in G.edges:\n",
+    "    x0, y0 = pos[u]\n",
+    "    x1, y1 = pos[v]\n",
+    "    edge_x.extend([x0, x1, None])\n",
+    "    edge_y.extend([y0, y1, None])\n",
+    "\n",
+    "edge_trace = go.Scatter(\n",
+    "    x=edge_x, y=edge_y,\n",
+    "    line=dict(width=0.5, color='#cccccc'),\n",
+    "    hoverinfo='none',\n",
+    "    mode='lines',\n",
+    "    name='Edges',\n",
+    ")\n",
+    "\n",
+    "print(f'Edge trace built with {len(G.edges)} edges')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Build node trace\n",
+    "node_x = []\n",
+    "node_y = []\n",
+    "node_text = []\n",
+    "node_color = []\n",
+    "node_size = []\n",
+    "\n",
+    "centralities = [G.nodes[n]['centrality'] for n in G.nodes]\n",
+    "max_centrality = max(centralities) if centralities else 1\n",
+    "\n",
+    "for node in G.nodes:\n",
+    "    x, y = pos[node]\n",
+    "    node_x.append(x)\n",
+    "    node_y.append(y)\n",
+    "\n",
+    "    data = G.nodes[node]\n",
+    "    degree = G.degree(node)\n",
+    "    hover_text = (\n",
+    "        f\"<b>{data['label']}</b><br>\"\n",
+    "        f\"Type: {data['type']}<br>\"\n",
+    "        f\"Degree: {degree}<br>\"\n",
+    "        f\"Community: {data['community']}<br>\"\n",
+    "        f\"Centrality: {data['centrality']:.6f}\"\n",
+    "    )\n",
+    "    node_text.append(hover_text)\n",
+    "    node_color.append(community_colors.get(data['community'], '#999999'))\n",
+    "\n",
+    "    # Node size proportional to centrality, with min/max bounds\n",
+    "    size = 5 + 30 * (data['centrality'] / max_centrality) if max_centrality > 0 else 10\n",
+    "    node_size.append(size)\n",
+    "\n",
+    "node_trace = go.Scatter(\n",
+    "    x=node_x, y=node_y,\n",
+    "    mode='markers',\n",
+    "    hoverinfo='text',\n",
+    "    hovertext=node_text,\n",
+    "    marker=dict(\n",
+    "        size=node_size,\n",
+    "        color=node_color,\n",
+    "        line=dict(width=1, color='#ffffff'),\n",
+    "    ),\n",
+    "    name='Entities',\n",
+    ")\n",
+    "\n",
+    "print(f'Node trace built with {G.number_of_nodes()} nodes')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create the figure\n",
+    "fig = go.Figure(\n",
+    "    data=[edge_trace, node_trace],\n",
+    "    layout=go.Layout(\n",
+    "        title=dict(\n",
+    "            text='Entity Co-occurrence Network',\n",
+    "            font=dict(size=20),\n",
+    "        ),\n",
+    "        showlegend=False,\n",
+    "        hovermode='closest',\n",
+    "        xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),\n",
+    "        yaxis=dict(showgrid=False, zeroline=False, showticklabels=False),\n",
+    "        plot_bgcolor='white',\n",
+    "        width=1200,\n",
+    "        height=800,\n",
+    "        margin=dict(l=20, r=20, t=60, b=20),\n",
+    "        annotations=[\n",
+    "            dict(\n",
+    "                text=f'Nodes: {G.number_of_nodes()} | Edges: {G.number_of_edges()} | Communities: {len(community_ids)}',\n",
+    "                xref='paper', yref='paper',\n",
+    "                x=0.5, y=-0.02,\n",
+    "                showarrow=False,\n",
+    "                font=dict(size=12, color='gray'),\n",
+    "            ),\n",
+    "        ],\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "fig.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Save as HTML for standalone viewing\n",
+    "output_path = save_fig(fig, 'entity_network_interactive', formats=('html',))\n",
+    "print(f'Saved interactive visualization to: {output_path}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Summary table: top entities\n",
+    "import pandas as pd\n",
+    "\n",
+    "node_stats = []\n",
+    "for node in G.nodes:\n",
+    "    data = G.nodes[node]\n",
+    "    node_stats.append({\n",
+    "        'entity': data['label'],\n",
+    "        'type': data['type'],\n",
+    "        'degree': G.degree(node),\n",
+    "        'community': data['community'],\n",
+    "        'centrality': data['centrality'],\n",
+    "    })\n",
+    "\n",
+    "stats_df = pd.DataFrame(node_stats).sort_values('centrality', ascending=False)\n",
+    "print('Top 30 entities by centrality:')\n",
+    "stats_df.head(30)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.10.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}