Refactor and enhance data import process for Supabase integration
Browse files- Improved function documentation and comments for clarity.
- Updated `create_table_if_not_exists` to accept DataFrame directly and added logging for SQL execution.
- Enhanced `import_data_to_supabase` to include a wait time for schema caching and improved error handling during data insertion.
- Modified Gradio interface labels for better user experience.
app.py
CHANGED
|
@@ -12,15 +12,18 @@ import supabase
|
|
| 12 |
# クライアントの初期化
|
| 13 |
supabase_client = supabase.create_client(SUPABASE_URL, SUPABASE_KEY)
|
| 14 |
|
| 15 |
-
import math
|
| 16 |
-
from io import StringIO
|
| 17 |
-
|
| 18 |
import gradio as gr
|
| 19 |
import pandas as pd
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
|
| 21 |
-
|
|
|
|
|
|
|
| 22 |
def pandas_dtype_to_supabase_type(dtype):
|
| 23 |
-
"""Maps pandas
|
| 24 |
if pd.api.types.is_integer_dtype(dtype):
|
| 25 |
return "integer"
|
| 26 |
elif pd.api.types.is_float_dtype(dtype):
|
|
@@ -32,14 +35,15 @@ def pandas_dtype_to_supabase_type(dtype):
|
|
| 32 |
else:
|
| 33 |
return "text"
|
| 34 |
|
| 35 |
-
|
| 36 |
-
|
|
|
|
|
|
|
| 37 |
"""
|
| 38 |
-
Supabase
|
| 39 |
-
'exec_sql(sql text)' という RPC を作成してある前提です。
|
| 40 |
"""
|
| 41 |
columns = []
|
| 42 |
-
for col_name, dtype in
|
| 43 |
supabase_type = pandas_dtype_to_supabase_type(dtype)
|
| 44 |
columns.append(f'"{col_name}" {supabase_type}')
|
| 45 |
|
|
@@ -50,9 +54,11 @@ def create_table_if_not_exists(db_name: str, df: pd.DataFrame):
|
|
| 50 |
{columns_sql}
|
| 51 |
);
|
| 52 |
"""
|
|
|
|
| 53 |
|
| 54 |
try:
|
| 55 |
response = supabase_client.rpc("exec_sql", {"sql": create_table_sql}).execute()
|
|
|
|
| 56 |
resp_data = getattr(response, "data", None)
|
| 57 |
resp_error = getattr(response, "error", None)
|
| 58 |
|
|
@@ -65,11 +71,17 @@ def create_table_if_not_exists(db_name: str, df: pd.DataFrame):
|
|
| 65 |
return True, f"Table '{db_name}' created or already exists. (no data returned)"
|
| 66 |
|
| 67 |
except Exception as e:
|
|
|
|
| 68 |
return False, f"Exception during table creation: {e}"
|
| 69 |
|
| 70 |
-
|
|
|
|
|
|
|
| 71 |
def clean_dataframe_for_json(df: pd.DataFrame):
|
| 72 |
-
"""
|
|
|
|
|
|
|
|
|
|
| 73 |
records = df.to_dict(orient="records")
|
| 74 |
cleaned = []
|
| 75 |
for row in records:
|
|
@@ -84,52 +96,51 @@ def clean_dataframe_for_json(df: pd.DataFrame):
|
|
| 84 |
cleaned.append(new_row)
|
| 85 |
return cleaned
|
| 86 |
|
| 87 |
-
|
|
|
|
|
|
|
| 88 |
def import_data_to_supabase(text_input: str, db_name: str):
|
| 89 |
-
|
| 90 |
-
タブ区切りの貼り付けテキストを DataFrame に変換し、
|
| 91 |
-
Supabase のテーブルへ挿入する。
|
| 92 |
-
"""
|
| 93 |
full_message = []
|
| 94 |
|
| 95 |
-
if not (text_input and text_input.strip()):
|
| 96 |
-
return "Please paste tab-separated data."
|
| 97 |
-
if not (db_name and db_name.strip()):
|
| 98 |
-
return "Please provide a table name."
|
| 99 |
-
|
| 100 |
try:
|
| 101 |
-
# 1
|
| 102 |
df = pd.read_csv(
|
| 103 |
StringIO(text_input),
|
| 104 |
sep="\t",
|
| 105 |
na_values=["null", "NULL", "NaN", ""],
|
| 106 |
-
keep_default_na=True
|
| 107 |
)
|
| 108 |
|
| 109 |
-
# 2
|
| 110 |
-
|
| 111 |
-
full_message.append(f"Table Creation Status: {
|
| 112 |
|
| 113 |
-
if not
|
| 114 |
return "\n".join(full_message)
|
| 115 |
|
| 116 |
-
# 3
|
|
|
|
|
|
|
|
|
|
| 117 |
data = clean_dataframe_for_json(df)
|
|
|
|
| 118 |
if not data:
|
| 119 |
-
full_message.append("Data Insertion Status:
|
| 120 |
return "\n".join(full_message)
|
| 121 |
|
| 122 |
-
#
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
resp_error = getattr(response, "error", None)
|
| 126 |
|
| 127 |
-
if
|
| 128 |
full_message.append(
|
| 129 |
-
f"Data Insertion Status: Error inserting data - {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
)
|
| 131 |
-
elif resp_data is not None:
|
| 132 |
-
full_message.append(f"Data Insertion Status: Successfully inserted {len(resp_data)} rows.")
|
| 133 |
else:
|
| 134 |
full_message.append("Data Insertion Status: Insert executed, but no data returned.")
|
| 135 |
|
|
@@ -138,22 +149,18 @@ def import_data_to_supabase(text_input: str, db_name: str):
|
|
| 138 |
|
| 139 |
return "\n".join(full_message)
|
| 140 |
|
| 141 |
-
|
| 142 |
# Gradio アプリ
|
|
|
|
| 143 |
iface = gr.Interface(
|
| 144 |
fn=import_data_to_supabase,
|
| 145 |
inputs=[
|
| 146 |
-
gr.Textbox(
|
| 147 |
-
|
| 148 |
-
lines=12,
|
| 149 |
-
placeholder="項目名\t下限\t上限\n曝気空気量(微小動物槽-1)\t11.3\t16.5\n…"
|
| 150 |
-
),
|
| 151 |
-
gr.Textbox(label="Supabase テーブル名(例: thresholds)"),
|
| 152 |
],
|
| 153 |
outputs="text",
|
| 154 |
-
title="Supabase Data Importer
|
| 155 |
-
description="
|
| 156 |
)
|
| 157 |
|
| 158 |
-
|
| 159 |
-
iface.launch()
|
|
|
|
| 12 |
# クライアントの初期化
|
| 13 |
supabase_client = supabase.create_client(SUPABASE_URL, SUPABASE_KEY)
|
| 14 |
|
|
|
|
|
|
|
|
|
|
| 15 |
import gradio as gr
|
| 16 |
import pandas as pd
|
| 17 |
+
import math
|
| 18 |
+
import time
|
| 19 |
+
from io import StringIO
|
| 20 |
+
import supabase
|
| 21 |
|
| 22 |
+
# =============================
|
| 23 |
+
# 型変換ヘルパー
|
| 24 |
+
# =============================
|
| 25 |
def pandas_dtype_to_supabase_type(dtype):
|
| 26 |
+
"""Maps pandas data types to Supabase SQL data types."""
|
| 27 |
if pd.api.types.is_integer_dtype(dtype):
|
| 28 |
return "integer"
|
| 29 |
elif pd.api.types.is_float_dtype(dtype):
|
|
|
|
| 35 |
else:
|
| 36 |
return "text"
|
| 37 |
|
| 38 |
+
# =============================
|
| 39 |
+
# テーブル作成関数
|
| 40 |
+
# =============================
|
| 41 |
+
def create_table_if_not_exists(db_name, dataframe):
|
| 42 |
"""
|
| 43 |
+
Creates a table in Supabase if it doesn't exist, using RPC `exec_sql`.
|
|
|
|
| 44 |
"""
|
| 45 |
columns = []
|
| 46 |
+
for col_name, dtype in dataframe.dtypes.items():
|
| 47 |
supabase_type = pandas_dtype_to_supabase_type(dtype)
|
| 48 |
columns.append(f'"{col_name}" {supabase_type}')
|
| 49 |
|
|
|
|
| 54 |
{columns_sql}
|
| 55 |
);
|
| 56 |
"""
|
| 57 |
+
print(f"Executing SQL for table creation:\n{create_table_sql}")
|
| 58 |
|
| 59 |
try:
|
| 60 |
response = supabase_client.rpc("exec_sql", {"sql": create_table_sql}).execute()
|
| 61 |
+
|
| 62 |
resp_data = getattr(response, "data", None)
|
| 63 |
resp_error = getattr(response, "error", None)
|
| 64 |
|
|
|
|
| 71 |
return True, f"Table '{db_name}' created or already exists. (no data returned)"
|
| 72 |
|
| 73 |
except Exception as e:
|
| 74 |
+
print(f"Exception during table creation RPC call: {e}")
|
| 75 |
return False, f"Exception during table creation: {e}"
|
| 76 |
|
| 77 |
+
# =============================
|
| 78 |
+
# DataFrame の JSON 変換ヘルパー
|
| 79 |
+
# =============================
|
| 80 |
def clean_dataframe_for_json(df: pd.DataFrame):
|
| 81 |
+
"""
|
| 82 |
+
Convert DataFrame to JSON-compliant list of dicts
|
| 83 |
+
(replace NaN, NaT, inf, -inf with None).
|
| 84 |
+
"""
|
| 85 |
records = df.to_dict(orient="records")
|
| 86 |
cleaned = []
|
| 87 |
for row in records:
|
|
|
|
| 96 |
cleaned.append(new_row)
|
| 97 |
return cleaned
|
| 98 |
|
| 99 |
+
# =============================
|
| 100 |
+
# メイン処理関数
|
| 101 |
+
# =============================
|
| 102 |
def import_data_to_supabase(text_input: str, db_name: str):
|
| 103 |
+
db_name = db_name.strip().lower() # 小文字に統一
|
|
|
|
|
|
|
|
|
|
| 104 |
full_message = []
|
| 105 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
try:
|
| 107 |
+
# 1. DataFrame 読み込み(タブ区切り & null対応)
|
| 108 |
df = pd.read_csv(
|
| 109 |
StringIO(text_input),
|
| 110 |
sep="\t",
|
| 111 |
na_values=["null", "NULL", "NaN", ""],
|
| 112 |
+
keep_default_na=True
|
| 113 |
)
|
| 114 |
|
| 115 |
+
# 2. テーブル作成
|
| 116 |
+
table_creation_success, table_creation_msg = create_table_if_not_exists(db_name, df)
|
| 117 |
+
full_message.append(f"Table Creation Status: {table_creation_msg}")
|
| 118 |
|
| 119 |
+
if not table_creation_success:
|
| 120 |
return "\n".join(full_message)
|
| 121 |
|
| 122 |
+
# 3. 数秒待機(スキーマキャッシュ反映待ち)
|
| 123 |
+
time.sleep(3)
|
| 124 |
+
|
| 125 |
+
# 4. JSON対応にクリーニング(NaN, NaT, inf → None)
|
| 126 |
data = clean_dataframe_for_json(df)
|
| 127 |
+
|
| 128 |
if not data:
|
| 129 |
+
full_message.append("Data Insertion Status: No data to insert.")
|
| 130 |
return "\n".join(full_message)
|
| 131 |
|
| 132 |
+
# 5. データ挿入
|
| 133 |
+
print(f"Attempting to insert {len(data)} rows into table '{db_name}'.")
|
| 134 |
+
response = supabase_client.table(db_name).insert(data).execute()
|
|
|
|
| 135 |
|
| 136 |
+
if getattr(response, "error", None):
|
| 137 |
full_message.append(
|
| 138 |
+
f"Data Insertion Status: Error inserting data - {response.error}"
|
| 139 |
+
)
|
| 140 |
+
elif getattr(response, "data", None):
|
| 141 |
+
full_message.append(
|
| 142 |
+
f"Data Insertion Status: Successfully inserted {len(response.data)} rows."
|
| 143 |
)
|
|
|
|
|
|
|
| 144 |
else:
|
| 145 |
full_message.append("Data Insertion Status: Insert executed, but no data returned.")
|
| 146 |
|
|
|
|
| 149 |
|
| 150 |
return "\n".join(full_message)
|
| 151 |
|
| 152 |
+
# =============================
|
| 153 |
# Gradio アプリ
|
| 154 |
+
# =============================
|
| 155 |
iface = gr.Interface(
|
| 156 |
fn=import_data_to_supabase,
|
| 157 |
inputs=[
|
| 158 |
+
gr.Textbox(label="Paste TSV Data Here", lines=15, placeholder="ここにタブ区切りデータを貼り付け"),
|
| 159 |
+
gr.Textbox(label="Supabase Database Table Name")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
],
|
| 161 |
outputs="text",
|
| 162 |
+
title="Supabase Data Importer",
|
| 163 |
+
description="タブ区切りデータを貼り付け、Supabase データベースのテーブルにインポートします。テーブルが存在しない場合は新しく作成します。"
|
| 164 |
)
|
| 165 |
|
| 166 |
+
iface.launch()
|
|
|