Spaces:
Sleeping
Sleeping
Deploy Rachana Data Studio
Browse files- data_studio/app.py +3 -3
- data_studio/sources.py +14 -37
data_studio/app.py
CHANGED
|
@@ -134,9 +134,9 @@ def render_overview(db) -> None:
|
|
| 134 |
st.warning("No source configs seeded yet.")
|
| 135 |
|
| 136 |
|
| 137 |
-
def render_sources(db) -> None:
|
| 138 |
-
st.subheader("Source setup")
|
| 139 |
-
st.caption("Seed the default source configurations for the
|
| 140 |
if st.button("Seed default sources", type="primary"):
|
| 141 |
inserted = seed_default_sources(db)
|
| 142 |
st.success(f"Source config seed completed. New records inserted: {inserted}.")
|
|
|
|
| 134 |
st.warning("No source configs seeded yet.")
|
| 135 |
|
| 136 |
|
| 137 |
+
def render_sources(db) -> None:
|
| 138 |
+
st.subheader("Source setup")
|
| 139 |
+
st.caption("Seed the default source configurations for the active review queues.")
|
| 140 |
if st.button("Seed default sources", type="primary"):
|
| 141 |
inserted = seed_default_sources(db)
|
| 142 |
st.success(f"Source config seed completed. New records inserted: {inserted}.")
|
data_studio/sources.py
CHANGED
|
@@ -108,41 +108,17 @@ def default_source_configs() -> list[dict[str, Any]]:
|
|
| 108 |
"updated_at": now,
|
| 109 |
}
|
| 110 |
)
|
| 111 |
-
sources.append(
|
| 112 |
-
{
|
| 113 |
-
"dataset_key": "wikipedia_telugu",
|
| 114 |
-
"name": "Wikipedia Telugu",
|
| 115 |
-
"description": "Pure Telugu document review queue from Wikimedia Wikipedia Telugu.",
|
| 116 |
-
"queue_name": "pure_telugu",
|
| 117 |
-
"sample_type": "document",
|
| 118 |
-
"hf_dataset": "wikimedia/wikipedia",
|
| 119 |
-
"hf_config": "20231101.te",
|
| 120 |
-
"hf_split": "train",
|
| 121 |
-
"hf_data_dir": None,
|
| 122 |
-
"field_mapping": {
|
| 123 |
-
"source_record_id": "id",
|
| 124 |
-
"text": "text",
|
| 125 |
-
"doc_type": None,
|
| 126 |
-
"title": "title",
|
| 127 |
-
"url": "url",
|
| 128 |
-
},
|
| 129 |
-
"filters": {"min_chars": 30},
|
| 130 |
-
"enabled": True,
|
| 131 |
-
"created_at": now,
|
| 132 |
-
"updated_at": now,
|
| 133 |
-
}
|
| 134 |
-
)
|
| 135 |
|
| 136 |
return sources
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
def seed_default_sources(db: Database) -> int:
|
| 140 |
-
inserted = 0
|
| 141 |
-
for source in default_source_configs():
|
| 142 |
-
result = db["source_configs"].update_one(
|
| 143 |
-
{"dataset_key": source["dataset_key"]},
|
| 144 |
-
{
|
| 145 |
-
"$set": {
|
| 146 |
"name": source["name"],
|
| 147 |
"description": source["description"],
|
| 148 |
"queue_name": source["queue_name"],
|
|
@@ -159,10 +135,11 @@ def seed_default_sources(db: Database) -> int:
|
|
| 159 |
"$setOnInsert": {"created_at": source["created_at"]},
|
| 160 |
},
|
| 161 |
upsert=True,
|
| 162 |
-
)
|
| 163 |
-
if result.upserted_id is not None:
|
| 164 |
-
inserted += 1
|
| 165 |
-
|
|
|
|
| 166 |
|
| 167 |
|
| 168 |
def _serialize_source(source: dict[str, Any]) -> dict[str, Any]:
|
|
|
|
| 108 |
"updated_at": now,
|
| 109 |
}
|
| 110 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
|
| 112 |
return sources
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def seed_default_sources(db: Database) -> int:
|
| 116 |
+
inserted = 0
|
| 117 |
+
for source in default_source_configs():
|
| 118 |
+
result = db["source_configs"].update_one(
|
| 119 |
+
{"dataset_key": source["dataset_key"]},
|
| 120 |
+
{
|
| 121 |
+
"$set": {
|
| 122 |
"name": source["name"],
|
| 123 |
"description": source["description"],
|
| 124 |
"queue_name": source["queue_name"],
|
|
|
|
| 135 |
"$setOnInsert": {"created_at": source["created_at"]},
|
| 136 |
},
|
| 137 |
upsert=True,
|
| 138 |
+
)
|
| 139 |
+
if result.upserted_id is not None:
|
| 140 |
+
inserted += 1
|
| 141 |
+
db["source_configs"].delete_many({"dataset_key": "wikipedia_telugu"})
|
| 142 |
+
return inserted
|
| 143 |
|
| 144 |
|
| 145 |
def _serialize_source(source: dict[str, Any]) -> dict[str, Any]:
|