Duplicate from CalebCometML/kangas-demo

Co-authored-by: Caleb K <CalebCometML@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+datagrids/coco-500.datagrid filter=lfs diff=lfs merge=lfs -text
+datagrids/beans.datagrid filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.idea

Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.9
+
+COPY requirements.txt /
+RUN pip install -r /requirements.txt
+RUN apt-get -y update && apt-get -y install nginx
+
+WORKDIR /app
+
+COPY ./ /app
+COPY nginx.conf /etc/nginx/nginx.conf
+RUN chmod 777 /app/startup.sh
+RUN chmod -R 777 /etc/nginx
+RUN chmod -R 777 /var/lib/
+RUN chmod -R 777 /var/log/
+RUN mkdir /.cache
+RUN chmod -R 777 /.cache
+RUN chmod -R 777 /app/datagrids
+RUN chmod -R 777 /usr/local/lib/python3.9/site-packages/kangas/frontend/standalone/.next/
+CMD /app/startup.sh

README.md

@@ -0,0 +1,13 @@
+---
+title: Kangas Demo
+emoji: 📚
+colorFrom: pink
+colorTo: yellow
+sdk: docker
+app_port: 7860
+pinned: false
+license: openrail
+duplicated_from: CalebCometML/kangas-demo
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

datagrids/coco-500.datagrid

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff36d97d6935da95ac918b607465931a34e20ff232c80ad89faeecbc5923edcd
+size 92835840

nginx.conf

@@ -0,0 +1,86 @@
+pid /tmp/nginx.pid; 
+
+events { 
+  worker_connections 1024; 
+}
+
+http {
+
+  server {
+    listen 7860 default_server;
+    listen [::]:7860 default_server;
+
+    server_name _;
+
+    add_header 'Content-Security-Policy' 'upgrade-insecure-requests';
+
+    location /kangas/ {
+        proxy_pass http://localhost:7640/;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection 'Upgrade';
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_cache_bypass $http_upgrade;
+        proxy_read_timeout 86400;
+        proxy_redirect off;
+        port_in_redirect off;
+    }
+
+    location /kangas_images/ {
+        proxy_pass http://localhost:7640/kangas_images/;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection 'Upgrade';
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_cache_bypass $http_upgrade;
+        proxy_read_timeout 86400;
+        proxy_redirect off;
+        port_in_redirect off;
+    }
+
+    location /_next/ {
+        proxy_pass http://localhost:7640/_next/;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection 'Upgrade';
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_cache_bypass $http_upgrade;
+        proxy_read_timeout 86400;
+        proxy_redirect off;
+    }
+
+    location /api/ {
+        proxy_pass http://localhost:7640/api/;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection 'Upgrade';
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_cache_bypass $http_upgrade;
+        proxy_read_timeout 86400;
+        proxy_redirect off;
+    }
+
+    location / {
+        proxy_pass http://localhost:7840/;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection 'Upgrade';
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_cache_bypass $http_upgrade;
+        proxy_read_timeout 86400;
+        proxy_redirect off;
+    }
+
+    location /healthCheck {
+        return 200 "OK!";
+    }
+
+  }
+}

notebooks/.ipynb_checkpoints/explore-checkpoint.ipynb

@@ -0,0 +1,137 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from IPython.core.interactiveshell import InteractiveShell\n",
+    "InteractiveShell.ast_node_interactivity = \"all\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "is_executing": true
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "28635666517e4f4c9e2889c7a3bc10db",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading builder script:   0%|          | 0.00/3.61k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0f9ccb664aa64f359bf15685ef3f5811",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading metadata:   0%|          | 0.00/2.24k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "efd478d8dcab433e8e8d802b4b23ec27",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading readme:   0%|          | 0.00/4.75k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Downloading and preparing dataset beans/default to /Users/derekthomas/.cache/huggingface/datasets/beans/default/0.0.0/90c755fb6db1c0ccdad02e897a37969dbf070bed3755d4391e269ff70642d791...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ec2ad1d1e6fe4be79d5b6226170250a3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading data files:   0%|          | 0/3 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4235b24184dc422ab70ef502e81c4b99",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading data:   0%|          | 0.00/144M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import kangas as kg\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "dataset = load_dataset(\"beans\", split=\"train\")\n",
+    "dg = kg.DataGrid(dataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

notebooks/explore.ipynb

@@ -0,0 +1,112 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from IPython.core.interactiveshell import InteractiveShell\n",
+    "InteractiveShell.ast_node_interactivity = \"all\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "proj_dir = Path.cwd().parent\n",
+    "proj_dir"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "pycharm": {
+     "is_executing": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import kangas as kg\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "# dataset = load_dataset(\"beans\", split=\"train\")\n",
+    "# dg = kg.DataGrid(dataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "        <iframe\n",
+       "            width=\"100%\"\n",
+       "            height=\"750px\"\n",
+       "            src=\"http://127.0.0.1:4000/\"\n",
+       "            frameborder=\"0\"\n",
+       "            allowfullscreen\n",
+       "            \n",
+       "        ></iframe>\n",
+       "        "
+      ],
+      "text/plain": [
+       "<IPython.lib.display.IFrame at 0x16d9231c0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "proj_dir = Path.cwd().parent\n",
+    "\n",
+    "import kangas as kg\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "dataset_repo = 'beans'\n",
+    "dataset = load_dataset(dataset_repo, split=\"train\")\n",
+    "dg = kg.DataGrid(dataset)\n",
+    "dg_file_name = dataset_repo.replace('/', '__') + '.datagrid' + '.2'\n",
+    "dg.save(proj_dir / 'datagrids' / dg_file_name)\n",
+    "kg.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

requirements.txt

@@ -0,0 +1,4 @@
+kangas==2.2.6
+datasets==2.10.1
+streamlit==1.20
+flask==2.2.3

startup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+kangas server --host=0.0.0.0 --frontend-port=7640 --frontend-root=kangas --debug &
+  
+streamlit run streamlitapp.py --server.port 7840 &
+
+nginx -g 'daemon off;'

streamlitapp.py

@@ -0,0 +1,32 @@
+from functools import partial
+from pathlib import Path
+
+import kangas as kg
+import streamlit as st
+import streamlit.components.v1 as components
+from datasets import load_dataset
+import socket
+
+proj_dir = Path(__file__).parent
+
+servername = 'calebcometml-kangas-demo.hf.space'
+src = f"https://{servername}/kangas/?datagrid=.%2Fdatagrids%2Fcoco-500.datagrid"
+
+st.set_page_config(layout="wide")
+
+st.markdown("1. Select dataset of your choice")
+
+def kangas_fn(dataset_repo):
+    repo_wo_slash = dataset_repo.replace('/', '__') + '.datagrid'
+    dg_file_name = repo_wo_slash + '.datagrid'
+    with st.spinner("Loading Dataset..."):
+        dataset = load_dataset(dataset_repo, split="train")
+    with st.spinner("Creating Kangas..."):
+        dg = kg.DataGrid(dataset)
+    with st.spinner("Saving Kangas..."):
+        dg.save(str(proj_dir / 'datagrids' / dg_file_name))
+
+hf_dataset = st.text_input("HuggingFace Dataset", value='beans')
+st.button("Download and Run", on_click=partial(kangas_fn, hf_dataset))
+st.markdown("""Click the dropdown in Kangas to see pre-loaded datasets""")
+st.components.v1.iframe(src, None, 1000, scrolling=True)