Update app.py
Browse files
app.py
CHANGED
|
@@ -2,6 +2,7 @@ import streamlit as st
|
|
| 2 |
import json
|
| 3 |
import pandas as pd
|
| 4 |
import os
|
|
|
|
| 5 |
|
| 6 |
st.set_page_config(page_title="Dataset Builder", layout="wide")
|
| 7 |
st.title("π JSONL Dataset Editor")
|
|
@@ -17,20 +18,24 @@ def get_all_fields(data):
|
|
| 17 |
all_keys.update(record.keys())
|
| 18 |
return sorted(all_keys)
|
| 19 |
|
| 20 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
if "data" not in st.session_state:
|
| 22 |
st.session_state.data = []
|
| 23 |
if "all_fields" not in st.session_state:
|
| 24 |
st.session_state.all_fields = []
|
| 25 |
-
if "
|
| 26 |
-
st.session_state.
|
| 27 |
|
| 28 |
-
# --- Load from temp if
|
| 29 |
-
if
|
| 30 |
with open(TMP_FILE, "r", encoding="utf-8") as f:
|
| 31 |
st.session_state.data = [json.loads(line) for line in f]
|
| 32 |
st.session_state.all_fields = get_all_fields(st.session_state.data)
|
| 33 |
-
st.session_state.prev_data = st.session_state.data.copy()
|
| 34 |
|
| 35 |
# --- Upload JSONL ---
|
| 36 |
uploaded_file = st.file_uploader("Upload a JSONL file", type=["jsonl"])
|
|
@@ -38,27 +43,22 @@ if uploaded_file:
|
|
| 38 |
content = uploaded_file.read().decode("utf-8")
|
| 39 |
st.session_state.data = [json.loads(line) for line in content.strip().splitlines()]
|
| 40 |
st.session_state.all_fields = get_all_fields(st.session_state.data)
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
for item in st.session_state.data:
|
| 44 |
-
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
| 45 |
st.rerun()
|
| 46 |
|
| 47 |
-
# ---
|
| 48 |
if not st.session_state.all_fields:
|
| 49 |
st.session_state.all_fields = ["context", "question", "answer"]
|
| 50 |
|
| 51 |
-
# --- Edit
|
| 52 |
st.markdown("### βοΈ Edit Records")
|
| 53 |
df = pd.DataFrame(st.session_state.data)
|
| 54 |
df = df.reindex(columns=st.session_state.all_fields)
|
| 55 |
|
| 56 |
-
# Ensure fields are strings for editor
|
| 57 |
for field in st.session_state.all_fields:
|
| 58 |
-
|
| 59 |
-
df[field] = df[field].astype(str)
|
| 60 |
|
| 61 |
-
# TextAreas for longer fields
|
| 62 |
column_configs = {
|
| 63 |
field: (
|
| 64 |
st.column_config.TextColumn(label=field, width="large")
|
|
@@ -70,35 +70,31 @@ column_configs = {
|
|
| 70 |
|
| 71 |
edited_df = st.data_editor(
|
| 72 |
df,
|
|
|
|
| 73 |
use_container_width=True,
|
| 74 |
num_rows="dynamic",
|
| 75 |
column_config=column_configs,
|
| 76 |
-
key="editable_table",
|
| 77 |
)
|
| 78 |
|
| 79 |
-
#
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
st.session_state.data
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
| 87 |
-
st.toast("β
Auto-saved!", icon="πΎ")
|
| 88 |
|
| 89 |
-
# --- Add
|
| 90 |
st.markdown("### β Add New Entry")
|
| 91 |
-
with st.form("
|
| 92 |
-
|
| 93 |
for field in st.session_state.all_fields:
|
| 94 |
-
|
| 95 |
submitted = st.form_submit_button("Add Entry")
|
| 96 |
if submitted:
|
| 97 |
-
st.session_state.data.append(
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
for item in st.session_state.data:
|
| 101 |
-
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
| 102 |
st.success("β
New entry added!")
|
| 103 |
st.rerun()
|
| 104 |
|
|
@@ -116,7 +112,6 @@ export_path = st.text_input("Save path", value="./exports/exported_dataset.jsonl
|
|
| 116 |
|
| 117 |
col1, col2, col3 = st.columns(3)
|
| 118 |
|
| 119 |
-
# Export
|
| 120 |
with col1:
|
| 121 |
if st.button("π Export JSONL"):
|
| 122 |
os.makedirs(os.path.dirname(export_path), exist_ok=True)
|
|
@@ -131,7 +126,6 @@ with col1:
|
|
| 131 |
st.session_state.clear()
|
| 132 |
st.rerun()
|
| 133 |
|
| 134 |
-
# Download temp
|
| 135 |
with col2:
|
| 136 |
if os.path.exists(TMP_FILE):
|
| 137 |
with open(TMP_FILE, "r", encoding="utf-8") as f:
|
|
@@ -142,10 +136,7 @@ with col2:
|
|
| 142 |
file_name="session_dataset.jsonl",
|
| 143 |
mime="application/json",
|
| 144 |
)
|
| 145 |
-
else:
|
| 146 |
-
st.warning("β οΈ No temp file found.")
|
| 147 |
|
| 148 |
-
# Clear session
|
| 149 |
with col3:
|
| 150 |
if st.button("ποΈ Clear Session"):
|
| 151 |
if os.path.exists(TMP_FILE):
|
|
|
|
| 2 |
import json
|
| 3 |
import pandas as pd
|
| 4 |
import os
|
| 5 |
+
from uuid import uuid4
|
| 6 |
|
| 7 |
st.set_page_config(page_title="Dataset Builder", layout="wide")
|
| 8 |
st.title("π JSONL Dataset Editor")
|
|
|
|
| 18 |
all_keys.update(record.keys())
|
| 19 |
return sorted(all_keys)
|
| 20 |
|
| 21 |
+
def save_to_file():
|
| 22 |
+
with open(TMP_FILE, "w", encoding="utf-8") as f:
|
| 23 |
+
for item in st.session_state.data:
|
| 24 |
+
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
| 25 |
+
|
| 26 |
+
# --- Initialize state ---
|
| 27 |
if "data" not in st.session_state:
|
| 28 |
st.session_state.data = []
|
| 29 |
if "all_fields" not in st.session_state:
|
| 30 |
st.session_state.all_fields = []
|
| 31 |
+
if "edit_key" not in st.session_state:
|
| 32 |
+
st.session_state.edit_key = str(uuid4())
|
| 33 |
|
| 34 |
+
# --- Load from temp if file exists ---
|
| 35 |
+
if os.path.exists(TMP_FILE) and not st.session_state.data:
|
| 36 |
with open(TMP_FILE, "r", encoding="utf-8") as f:
|
| 37 |
st.session_state.data = [json.loads(line) for line in f]
|
| 38 |
st.session_state.all_fields = get_all_fields(st.session_state.data)
|
|
|
|
| 39 |
|
| 40 |
# --- Upload JSONL ---
|
| 41 |
uploaded_file = st.file_uploader("Upload a JSONL file", type=["jsonl"])
|
|
|
|
| 43 |
content = uploaded_file.read().decode("utf-8")
|
| 44 |
st.session_state.data = [json.loads(line) for line in content.strip().splitlines()]
|
| 45 |
st.session_state.all_fields = get_all_fields(st.session_state.data)
|
| 46 |
+
save_to_file()
|
| 47 |
+
st.session_state.edit_key = str(uuid4())
|
|
|
|
|
|
|
| 48 |
st.rerun()
|
| 49 |
|
| 50 |
+
# --- Ensure default fields if none ---
|
| 51 |
if not st.session_state.all_fields:
|
| 52 |
st.session_state.all_fields = ["context", "question", "answer"]
|
| 53 |
|
| 54 |
+
# --- Edit Section ---
|
| 55 |
st.markdown("### βοΈ Edit Records")
|
| 56 |
df = pd.DataFrame(st.session_state.data)
|
| 57 |
df = df.reindex(columns=st.session_state.all_fields)
|
| 58 |
|
|
|
|
| 59 |
for field in st.session_state.all_fields:
|
| 60 |
+
df[field] = df[field].astype(str)
|
|
|
|
| 61 |
|
|
|
|
| 62 |
column_configs = {
|
| 63 |
field: (
|
| 64 |
st.column_config.TextColumn(label=field, width="large")
|
|
|
|
| 70 |
|
| 71 |
edited_df = st.data_editor(
|
| 72 |
df,
|
| 73 |
+
key=st.session_state.edit_key, # Unique key forces Streamlit to rerender editor
|
| 74 |
use_container_width=True,
|
| 75 |
num_rows="dynamic",
|
| 76 |
column_config=column_configs,
|
|
|
|
| 77 |
)
|
| 78 |
|
| 79 |
+
# If data changed, update state and save
|
| 80 |
+
if edited_df is not None:
|
| 81 |
+
new_data = edited_df.fillna("").to_dict(orient="records")
|
| 82 |
+
if new_data != st.session_state.data:
|
| 83 |
+
st.session_state.data = new_data
|
| 84 |
+
save_to_file()
|
| 85 |
+
st.toast("β
Changes auto-saved!", icon="πΎ")
|
|
|
|
|
|
|
| 86 |
|
| 87 |
+
# --- Add Entry ---
|
| 88 |
st.markdown("### β Add New Entry")
|
| 89 |
+
with st.form("add_form"):
|
| 90 |
+
new_item = {}
|
| 91 |
for field in st.session_state.all_fields:
|
| 92 |
+
new_item[field] = st.text_area(field, key=f"add_{field}")
|
| 93 |
submitted = st.form_submit_button("Add Entry")
|
| 94 |
if submitted:
|
| 95 |
+
st.session_state.data.append(new_item)
|
| 96 |
+
save_to_file()
|
| 97 |
+
st.session_state.edit_key = str(uuid4()) # Force editor to refresh
|
|
|
|
|
|
|
| 98 |
st.success("β
New entry added!")
|
| 99 |
st.rerun()
|
| 100 |
|
|
|
|
| 112 |
|
| 113 |
col1, col2, col3 = st.columns(3)
|
| 114 |
|
|
|
|
| 115 |
with col1:
|
| 116 |
if st.button("π Export JSONL"):
|
| 117 |
os.makedirs(os.path.dirname(export_path), exist_ok=True)
|
|
|
|
| 126 |
st.session_state.clear()
|
| 127 |
st.rerun()
|
| 128 |
|
|
|
|
| 129 |
with col2:
|
| 130 |
if os.path.exists(TMP_FILE):
|
| 131 |
with open(TMP_FILE, "r", encoding="utf-8") as f:
|
|
|
|
| 136 |
file_name="session_dataset.jsonl",
|
| 137 |
mime="application/json",
|
| 138 |
)
|
|
|
|
|
|
|
| 139 |
|
|
|
|
| 140 |
with col3:
|
| 141 |
if st.button("ποΈ Clear Session"):
|
| 142 |
if os.path.exists(TMP_FILE):
|