KPrashanth commited on
Commit
d987c5c
·
verified ·
1 Parent(s): 684e89d

Deploy Rachana Data Studio

Browse files
Files changed (2) hide show
  1. data_studio/app.py +3 -3
  2. data_studio/sources.py +14 -37
data_studio/app.py CHANGED
@@ -134,9 +134,9 @@ def render_overview(db) -> None:
134
  st.warning("No source configs seeded yet.")
135
 
136
 
137
- def render_sources(db) -> None:
138
- st.subheader("Source setup")
139
- st.caption("Seed the default source configurations for the first three review queues.")
140
  if st.button("Seed default sources", type="primary"):
141
  inserted = seed_default_sources(db)
142
  st.success(f"Source config seed completed. New records inserted: {inserted}.")
 
134
  st.warning("No source configs seeded yet.")
135
 
136
 
137
+ def render_sources(db) -> None:
138
+ st.subheader("Source setup")
139
+ st.caption("Seed the default source configurations for the active review queues.")
140
  if st.button("Seed default sources", type="primary"):
141
  inserted = seed_default_sources(db)
142
  st.success(f"Source config seed completed. New records inserted: {inserted}.")
data_studio/sources.py CHANGED
@@ -108,41 +108,17 @@ def default_source_configs() -> list[dict[str, Any]]:
108
  "updated_at": now,
109
  }
110
  )
111
- sources.append(
112
- {
113
- "dataset_key": "wikipedia_telugu",
114
- "name": "Wikipedia Telugu",
115
- "description": "Pure Telugu document review queue from Wikimedia Wikipedia Telugu.",
116
- "queue_name": "pure_telugu",
117
- "sample_type": "document",
118
- "hf_dataset": "wikimedia/wikipedia",
119
- "hf_config": "20231101.te",
120
- "hf_split": "train",
121
- "hf_data_dir": None,
122
- "field_mapping": {
123
- "source_record_id": "id",
124
- "text": "text",
125
- "doc_type": None,
126
- "title": "title",
127
- "url": "url",
128
- },
129
- "filters": {"min_chars": 30},
130
- "enabled": True,
131
- "created_at": now,
132
- "updated_at": now,
133
- }
134
- )
135
 
136
  return sources
137
-
138
-
139
- def seed_default_sources(db: Database) -> int:
140
- inserted = 0
141
- for source in default_source_configs():
142
- result = db["source_configs"].update_one(
143
- {"dataset_key": source["dataset_key"]},
144
- {
145
- "$set": {
146
  "name": source["name"],
147
  "description": source["description"],
148
  "queue_name": source["queue_name"],
@@ -159,10 +135,11 @@ def seed_default_sources(db: Database) -> int:
159
  "$setOnInsert": {"created_at": source["created_at"]},
160
  },
161
  upsert=True,
162
- )
163
- if result.upserted_id is not None:
164
- inserted += 1
165
- return inserted
 
166
 
167
 
168
  def _serialize_source(source: dict[str, Any]) -> dict[str, Any]:
 
108
  "updated_at": now,
109
  }
110
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
111
 
112
  return sources
113
+
114
+
115
+ def seed_default_sources(db: Database) -> int:
116
+ inserted = 0
117
+ for source in default_source_configs():
118
+ result = db["source_configs"].update_one(
119
+ {"dataset_key": source["dataset_key"]},
120
+ {
121
+ "$set": {
122
  "name": source["name"],
123
  "description": source["description"],
124
  "queue_name": source["queue_name"],
 
135
  "$setOnInsert": {"created_at": source["created_at"]},
136
  },
137
  upsert=True,
138
+ )
139
+ if result.upserted_id is not None:
140
+ inserted += 1
141
+ db["source_configs"].delete_many({"dataset_key": "wikipedia_telugu"})
142
+ return inserted
143
 
144
 
145
  def _serialize_source(source: dict[str, Any]) -> dict[str, Any]: