ashokpoudel commited on
Commit
0ff5b4f
·
verified ·
1 Parent(s): b0aa85d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +414 -418
app.py CHANGED
@@ -6,8 +6,10 @@ import random
6
  import datetime
7
  import uuid
8
  import json
9
- from huggingface_hub import HfApi
10
  from datasets import Dataset
 
 
11
 
12
  # Configuration
13
  SAMPLE_PROMPTS = [
@@ -38,7 +40,6 @@ REGIONS = [
38
  "सुदूरपश्चिम प्रदेश (Sudurpashchim Province)"
39
  ]
40
 
41
- # Common last names by ethnicity/region for better accent tracking
42
  COMMON_LAST_NAMES = {
43
  "पहाडी (Pahadi)": ["शर्मा (Sharma)", "पौडेल (Poudel)", "खनाल (Khanal)", "अधिकारी (Adhikari)", "भट्टराई (Bhattarai)", "अन्य पहाडी (Other Pahadi)"],
44
  "नेवार (Newar)": ["श्रेष्ठ (Shrestha)", "प्रधान (Pradhan)", "महर्जन (Maharjan)", "बज्राचार्य (Bajracharya)", "अन्य नेवार (Other Newar)"],
@@ -53,52 +54,63 @@ COMMON_LAST_NAMES = {
53
  "अन्य (Other)": ["अन्य (Other)"]
54
  }
55
 
56
- # Create directory for saved recordings
57
- os.makedirs("recordings", exist_ok=True)
58
- os.makedirs("metadata", exist_ok=True)
59
- os.makedirs("ratings", exist_ok=True)
 
 
 
 
60
 
61
- # Initialize metadata file if it doesn't exist
62
- metadata_file = "metadata/metadata.csv"
63
- ratings_file = "ratings/ratings.json"
 
 
 
64
 
65
- if not os.path.exists(metadata_file):
66
- pd.DataFrame(columns=[
67
- "id", "text", "audio_path", "gender", "age_group", "ethnicity",
68
- "last_name", "region", "emotion", "timestamp", "recording_type"
69
- ]).to_csv(metadata_file, index=False)
70
 
71
- if not os.path.exists(ratings_file):
72
- with open(ratings_file, 'w') as f:
73
- json.dump({}, f)
74
 
 
 
 
75
  def save_recording(audio, text, gender, age_group, ethnicity, last_name, region, emotion, recording_type):
76
  """Save the recording and metadata"""
77
- # Generate unique ID for this recording
78
- recording_id = str(uuid.uuid4())
79
- timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
80
-
81
- # Check if audio was recorded
82
  if audio is None:
83
  return "कृपया पहिले रेकर्डिङ गर्नुहोस्। (Please record audio first)", None
84
-
85
- # Save audio file
86
- audio_filename = f"recordings/{recording_id}.wav"
87
- if isinstance(audio, tuple): # If it's a tuple (sr, data)
88
- sr, data = audio
89
- import soundfile as sf
90
- sf.write(audio_filename, data, sr)
91
- else: # If it's a path
92
- # Copy the file
93
- import shutil
94
- shutil.copy(audio, audio_filename)
95
-
96
- # Update metadata
97
- metadata = pd.read_csv(metadata_file)
 
 
 
 
 
 
98
  new_row = pd.DataFrame([{
99
  "id": recording_id,
100
  "text": text,
101
- "audio_path": audio_filename,
102
  "gender": gender,
103
  "age_group": age_group,
104
  "ethnicity": ethnicity,
@@ -108,438 +120,422 @@ def save_recording(audio, text, gender, age_group, ethnicity, last_name, region,
108
  "timestamp": timestamp,
109
  "recording_type": recording_type
110
  }])
111
-
112
- updated_metadata = pd.concat([metadata, new_row], ignore_index=True)
113
- updated_metadata.to_csv(metadata_file, index=False)
114
-
115
- # Initialize rating for this recording in the ratings file
116
- with open(ratings_file, 'r') as f:
117
  ratings = json.load(f)
118
-
119
- ratings[recording_id] = {
120
- "upvotes": 0,
121
- "downvotes": 0,
122
- "quality_score": 0, # Average quality rating (1-5)
123
- "quality_votes": 0, # Number of quality ratings
124
- "correctness_score": 0, # Average correctness rating (1-5)
125
- "correctness_votes": 0 # Number of correctness ratings
126
- }
127
-
128
- with open(ratings_file, 'w') as f:
129
  json.dump(ratings, f, indent=2)
130
-
131
- return f"रेकर्डिङ सफलतापूर्वक सुरक्षित गरियो! (Recording saved successfully!)", audio_filename
 
132
 
133
  def get_random_prompt():
134
- """Return a random prompt from the list"""
135
  return random.choice(SAMPLE_PROMPTS)
136
 
137
- def vote_recording(recording_id, vote_type, vote_value):
138
- """Add a vote for a recording"""
139
- if not os.path.exists(ratings_file):
 
 
 
 
140
  return "रेटिङ फाइल भेटिएन। (Rating file not found.)"
141
-
142
  try:
143
- with open(ratings_file, 'r') as f:
 
 
 
 
 
 
 
144
  ratings = json.load(f)
145
-
146
- if recording_id not in ratings:
147
- return "रेकर्डिङ आईडी भेटिएन। (Recording ID not found.)"
148
-
149
- if vote_type == "upvote":
150
- ratings[recording_id]["upvotes"] += 1
151
- elif vote_type == "downvote":
152
- ratings[recording_id]["downvotes"] += 1
153
- elif vote_type == "quality":
154
- # Update quality score (running average)
155
- current_score = ratings[recording_id]["quality_score"]
156
- current_votes = ratings[recording_id]["quality_votes"]
157
- new_votes = current_votes + 1
158
- new_score = ((current_score * current_votes) + vote_value) / new_votes
159
-
160
- ratings[recording_id]["quality_score"] = new_score
161
- ratings[recording_id]["quality_votes"] = new_votes
162
- elif vote_type == "correctness":
163
- # Update correctness score (running average)
164
- current_score = ratings[recording_id]["correctness_score"]
165
- current_votes = ratings[recording_id]["correctness_votes"]
166
- new_votes = current_votes + 1
167
- new_score = ((current_score * current_votes) + vote_value) / new_votes
168
-
169
- ratings[recording_id]["correctness_score"] = new_score
170
- ratings[recording_id]["correctness_votes"] = new_votes
171
-
172
- with open(ratings_file, 'w') as f:
173
  json.dump(ratings, f, indent=2)
174
-
175
- return f"मतदान सफलतापूर्वक दर्ता गरियो! (Vote registered successfully!)"
176
-
177
  except Exception as e:
178
- return f"त्रुटि: {str(e)}"
179
-
180
- def get_ethnicity_based_last_names(ethnicity):
181
- """Return last name options based on selected ethnicity"""
182
- if ethnicity in COMMON_LAST_NAMES:
183
- return COMMON_LAST_NAMES[ethnicity]
184
- return COMMON_LAST_NAMES["अन्य (Other)"]
185
 
186
- def upload_to_huggingface(hf_token, dataset_name):
187
  """Upload the collected data to Hugging Face"""
188
- if not os.path.exists(metadata_file):
189
- return "कुनै डाटा भेटिएन। (No data found.)"
190
-
 
 
 
 
 
 
 
 
 
 
 
 
 
191
  try:
192
- # Read metadata
193
- metadata = pd.read_csv(metadata_file)
194
-
 
 
195
  if len(metadata) == 0:
196
- return "कुनै डाटा भेटिएन। (No data found.)"
197
-
198
- # Read ratings
199
- with open(ratings_file, 'r') as f:
200
- ratings = json.load(f)
201
-
202
- # Add ratings to metadata
203
- metadata["upvotes"] = metadata["id"].apply(lambda x: ratings.get(x, {}).get("upvotes", 0))
204
- metadata["downvotes"] = metadata["id"].apply(lambda x: ratings.get(x, {}).get("downvotes", 0))
205
- metadata["quality_score"] = metadata["id"].apply(lambda x: ratings.get(x, {}).get("quality_score", 0))
206
- metadata["correctness_score"] = metadata["id"].apply(lambda x: ratings.get(x, {}).get("correctness_score", 0))
 
 
 
 
 
 
 
 
 
207
 
208
- # Create a dataset dict
209
- dataset_dict = {
210
- "id": metadata["id"].tolist(),
211
- "text": metadata["text"].tolist(),
212
- "gender": metadata["gender"].tolist(),
213
- "age_group": metadata["age_group"].tolist(),
214
- "ethnicity": metadata["ethnicity"].tolist(),
215
- "last_name": metadata["last_name"].tolist(),
216
- "region": metadata["region"].tolist(),
217
- "emotion": metadata["emotion"].tolist(),
218
- "recording_type": metadata["recording_type"].tolist(),
219
- "timestamp": metadata["timestamp"].tolist(),
220
- "upvotes": metadata["upvotes"].tolist(),
221
- "downvotes": metadata["downvotes"].tolist(),
222
- "quality_score": metadata["quality_score"].tolist(),
223
- "correctness_score": metadata["correctness_score"].tolist(),
224
- }
225
 
226
- # Create a Dataset object
227
- dataset = Dataset.from_dict(dataset_dict)
 
 
 
228
 
229
- # Push to hub
230
- api = HfApi(token=hf_token)
231
- dataset.push_to_hub(dataset_name)
 
 
 
 
 
 
 
 
 
 
 
 
232
 
233
- # Upload audio files
234
  for _, row in metadata.iterrows():
235
- audio_path = row["audio_path"]
236
- if os.path.exists(audio_path):
 
 
237
  api.upload_file(
238
- path_or_fileobj=audio_path,
239
- path_in_repo=f"audio/{os.path.basename(audio_path)}",
240
  repo_id=dataset_name,
241
  repo_type="dataset"
242
  )
243
-
244
- return f"डाटा हगिङफेसमा सफलतापूर्वक अपलोड गरियो! (Data successfully uploaded to Hugging Face at {dataset_name})"
245
-
 
 
246
  except Exception as e:
247
- return f"त्रुटि: {str(e)}"
 
 
248
 
249
  def update_count():
250
- """Update the count of recordings"""
251
- if os.path.exists(metadata_file):
252
- metadata = pd.read_csv(metadata_file)
253
- return f"���ालसम्म {len(metadata)} रेकर्डिङहरू संकलन गरिएको छ। (Total recordings collected: {len(metadata)})"
 
 
254
  return "कुनै रेकर्डिङ भेटिएन। (No recordings found.)"
255
 
256
  def list_recordings(num_items=10):
257
- """List recent recordings for review"""
258
- if not os.path.exists(metadata_file):
259
- return pd.DataFrame()
260
-
261
- metadata = pd.read_csv(metadata_file)
 
 
262
  if len(metadata) == 0:
263
- return pd.DataFrame()
264
-
265
- # Sort by timestamp (newest first) and get the most recent entries
266
- metadata['timestamp'] = pd.to_datetime(metadata['timestamp'])
267
- sorted_metadata = metadata.sort_values('timestamp', ascending=False).head(num_items)
268
-
269
- # Reset the index for display purposes
270
  display_df = sorted_metadata[['id', 'text', 'ethnicity', 'region', 'timestamp']].copy()
271
- display_df['timestamp'] = display_df['timestamp'].dt.strftime('%Y-%m-%d %H:%M')
272
- display_df = display_df.reset_index(drop=True)
273
-
274
- return display_df
275
 
276
  def get_recording_audio(recording_id):
277
- """Get audio file path for a specific recording"""
278
- if not os.path.exists(metadata_file):
279
- return None, "रेकर्डिङ भेटिएन। (Recording not found.)"
280
-
281
- metadata = pd.read_csv(metadata_file)
 
 
282
  recording = metadata[metadata['id'] == recording_id]
283
-
284
- if len(recording) == 0:
285
- return None, "रेकर्डिङ भेटिएन। (Recording not found.)"
286
-
287
  audio_path = recording['audio_path'].iloc[0]
288
  text = recording['text'].iloc[0]
289
-
290
- if not os.path.exists(audio_path):
291
- return None, "अडियो फाइल भेटिएन। (Audio file not found.)"
292
-
293
  return audio_path, text
294
 
295
  def get_recording_ratings(recording_id):
296
- """Get current ratings for a recording"""
297
- if not os.path.exists(ratings_file):
298
- return "डाटा भेटिएन। (No data found.)"
299
-
300
- with open(ratings_file, 'r') as f:
301
  ratings = json.load(f)
302
-
303
- if recording_id not in ratings:
304
- return "रेकर्डिङ आईडी भेटिएन। (Recording ID not found.)"
305
-
306
  r = ratings[recording_id]
307
-
308
- # Format the ratings for display
309
- upvotes = r["upvotes"]
310
- downvotes = r["downvotes"]
311
- quality = round(r["quality_score"], 1) if r["quality_votes"] > 0 else 0
312
- quality_votes = r["quality_votes"]
313
- correctness = round(r["correctness_score"], 1) if r["correctness_votes"] > 0 else 0
314
- correctness_votes = r["correctness_votes"]
315
-
316
  return f"""👍 Upvotes: {upvotes} | 👎 Downvotes: {downvotes}
317
  गुणस्तर (Quality): {quality}/5 ({quality_votes} मत/votes)
318
  शुद्धता (Correctness): {correctness}/5 ({correctness_votes} मत/votes)"""
319
 
 
320
  def build_ui():
321
- """Build the Gradio interface"""
322
  with gr.Blocks(title="नेपाली ASR डाटा संकलन (Nepali ASR Data Collection)") as app:
323
- gr.Markdown("""
324
- # नेपाली ASR डाटा संकलन (Nepali ASR Data Collection)
325
-
326
- यस प्लेटफर्मले नेपाली भाषाको स्वचालित भाषण पहिचान (ASR) प्रविधिको विकासका लागि आवाज डाटा संकलन गर्दछ।
327
- कृपया आफ्नो आवाज रेकर्ड गरेर योगदान दिनुहोस्।
328
-
329
- *This platform collects voice data for the development of Nepali Automatic Speech Recognition (ASR) technology.
330
- Please contribute by recording your voice.*
331
- """)
332
-
333
- with gr.Tab("स्वतन्त्र पाठ (Free Text)"):
334
- with gr.Row():
335
- with gr.Column():
336
- free_text = gr.Textbox(
337
- label="तपाईंले बोल्न चाहनुभएको पाठ यहाँ लेख्नुहोस् (Type the text you want to speak here)",
338
- placeholder="यहाँ लेख्नुहोस्...",
339
- lines=3
340
- )
341
- free_audio = gr.Audio(
342
- label="आफ्नो आवाज रेकर्ड गर्नुहोस् (Record your voice)",
343
- type="filepath",
344
- source="microphone"
345
- )
346
-
347
- with gr.Column():
348
- # First row of metadata
349
- with gr.Row():
350
- free_gender = gr.Dropdown(
351
- label="लिङ्ग (Gender)",
352
- choices=GENDERS,
353
- value=GENDERS[0]
354
- )
355
- free_age = gr.Dropdown(
356
- label="उमेर समूह (Age Group)",
357
- choices=AGE_GROUPS,
358
- value=AGE_GROUPS[1]
359
- )
360
-
361
- # Second row of metadata
362
- with gr.Row():
363
- free_ethnicity = gr.Dropdown(
364
- label="जातीयता (Ethnicity)",
365
- choices=list(COMMON_LAST_NAMES.keys()),
366
- value=list(COMMON_LAST_NAMES.keys())[0]
367
- )
368
- free_last_name = gr.Dropdown(
369
- label="थर (Last Name)",
370
- choices=COMMON_LAST_NAMES[list(COMMON_LAST_NAMES.keys())[0]]
371
- )
372
-
373
- # Update last name options when ethnicity changes
374
- free_ethnicity.change(
375
- fn=get_ethnicity_based_last_names,
376
- inputs=free_ethnicity,
377
- outputs=free_last_name
378
- )
379
-
380
- # Third row of metadata
381
- with gr.Row():
382
- free_region = gr.Dropdown(
383
- label="क्षेत्र (Region)",
384
- choices=REGIONS,
385
- value=REGIONS[2] # Default to Bagmati Province
386
  )
387
- free_emotion = gr.Dropdown(
388
- label="भावना (Emotion)",
389
- choices=EMOTIONS,
390
- value=EMOTIONS[0]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
391
  )
392
-
393
- free_submit = gr.Button("सुरक्षित गर्नुहोस् (Save)")
394
- free_output = gr.Textbox(label="स्थिति (Status)")
395
-
396
- free_submit.click(
397
- fn=save_recording,
398
- inputs=[
399
- free_audio, free_text, free_gender, free_age,
400
- free_ethnicity, free_last_name, free_region, free_emotion,
401
- gr.Textbox(value="free_text", visible=False)
402
- ],
403
- outputs=[free_output, free_audio]
404
- )="filepath",
405
- source="microphone"
406
- )
407
-
408
- with gr.Column():
409
- free_gender = gr.Dropdown(
410
- label="लिङ्ग (Gender)",
411
- choices=GENDERS,
412
- value=GENDERS[0]
413
- )
414
- free_age = gr.Dropdown(
415
- label="उमेर समूह (Age Group)",
416
- choices=AGE_GROUPS,
417
- value=AGE_GROUPS[1]
418
- )
419
- free_emotion = gr.Dropdown(
420
- label="भावना (Emotion)",
421
- choices=EMOTIONS,
422
- value=EMOTIONS[0]
423
- )
424
-
425
- free_submit = gr.Button("सुरक्षित गर्नुहोस् (Save)")
426
- free_output = gr.Textbox(label="स्थिति (Status)")
427
-
428
- free_submit.click(
429
- fn=save_recording,
430
- inputs=[free_audio, free_text, free_gender, free_age, free_emotion, gr.Textbox(value="free_text", visible=False)],
431
- outputs=[free_output, free_audio]
432
- )
433
-
434
- with gr.Tab("निर्देशित पाठ (Prompted Text)"):
435
- with gr.Row():
436
- with gr.Column():
437
- prompt_text = gr.Textbox(
438
- label="कृपया यो पाठ पढ्नुहोस् (Please read this text)",
439
- value=get_random_prompt(),
440
- lines=3
441
- )
442
- prompt_audio = gr.Audio(
443
- label="आफ्नो आवाज रेकर्ड गर्नुहोस् (Record your voice)",
444
- type="filepath",
445
- source="microphone"
446
- )
447
- new_prompt = gr.Button("नयाँ पाठ (New Text)")
448
-
449
- with gr.Column():
450
- prompt_gender = gr.Dropdown(
451
- label="लिङ्ग (Gender)",
452
- choices=GENDERS,
453
- value=GENDERS[0]
454
- )
455
- prompt_age = gr.Dropdown(
456
- label="उमेर समूह (Age Group)",
457
- choices=AGE_GROUPS,
458
- value=AGE_GROUPS[1]
459
- )
460
- prompt_emotion = gr.Dropdown(
461
- label="भावना (Emotion)",
462
- choices=EMOTIONS,
463
- value=EMOTIONS[0]
464
- )
465
-
466
- prompt_submit = gr.Button("सुरक्षित गर्नुहोस् (Save)")
467
- prompt_output = gr.Textbox(label="स्थिति (Status)")
468
-
469
- new_prompt.click(fn=get_random_prompt, inputs=None, outputs=prompt_text)
470
- prompt_submit.click(
471
- fn=save_recording,
472
- inputs=[prompt_audio, prompt_text, prompt_gender, prompt_age, prompt_emotion, gr.Textbox(value="prompted_text", visible=False)],
473
- outputs=[prompt_output, prompt_audio]
474
- )
475
-
476
- with gr.Tab("प्रगति (Progress)"):
477
- count_display = gr.Textbox(label="संकलित रेकर्डिङ गणना (Recording Count)")
478
- refresh_button = gr.Button("ताजा गर्नुहोस् (Refresh)")
479
- refresh_button.click(fn=update_count, inputs=None, outputs=count_display)
480
-
481
- # HuggingFace upload section (admin only)
482
- gr.Markdown("## हगिङफेसमा अपलोड गर्नुहोस् (Upload to Hugging Face)")
483
- with gr.Row():
484
- hf_token = gr.Textbox(label="Hugging Face API Token", type="password")
485
- dataset_name = gr.Textbox(
486
- label="Dataset Name",
487
- placeholder="username/nepali-asr-dataset"
488
  )
489
- upload_button = gr.Button("अपलोड गर्नुहोस् (Upload)")
490
- upload_status = gr.Textbox(label="अपलोड स्थिति (Upload Status)")
491
-
492
- upload_button.click(
493
- fn=upload_to_huggingface,
494
- inputs=[hf_token, dataset_name],
495
- outputs=upload_status
496
- )
497
-
498
- with gr.Tab("जानकारी (Information)"):
499
- gr.Markdown("""
500
- ## नेपाली ASR डाटा संकलन प्रोजेक्टको बारेमा
501
-
502
- यो प्रोजेक्टले नेपाली भाषाको स्वचालित भाषण पहिचान (ASR) प्रविधिको विकासका लागि आवश्यक डाटा संकलन गर्दछ।
503
- तपाईंको योगदानले नेपाली भाषा प्रविधिको विकासमा ठूलो मद्दत पुर्‍याउनेछ।
504
-
505
- ### कसरी योगदान दिने:
506
-
507
- 1. **स्वतन्त्र पाठ (Free Text)** ट्याबमा, तपाईं आफ्नो इच्छा अनुसार पाठ लेखेर त्यसलाई बोल्न सक्नुहुन्छ।
508
- 2. **निर्देशित पाठ (Prompted Text)** ट्याबमा, तपाईंले दिइएको पाठलाई पढेर रेकर्ड गर्न सक्नुहुन्छ।
509
- 3. रेकर्डिङ पछि, "सुरक्षित गर्नुहोस्" बटनमा क्लिक गर्नुहोस्।
510
-
511
- ### गोपनीयता नीति:
512
-
513
- - तपाईंको आवाज रेकर्डिङ र मेटाडाटा सार्वजनिक अनुसन्धान उद्देश्यका लागि प्रयोग गरिनेछ।
514
- - कृपया व्यक्तिगत पहिचान गर्न सकिने जानकारी शेयर नगर्नुहोस्।
515
- - यो डाटासेट खुला स्रोत हुनेछ र हगिङफेसमा प्रकाशित गरिनेछ।
516
-
517
- ---
518
-
519
- ## About Nepali ASR Data Collection Project
520
-
521
- This project collects necessary data for the development of Nepali Automatic Speech Recognition (ASR) technology.
522
- Your contribution will greatly help in advancing Nepali language technology.
523
-
524
- ### How to Contribute:
525
-
526
- 1. In the **Free Text** tab, you can type any text you want and record yourself speaking it.
527
- 2. In the **Prompted Text** tab, you can record yourself reading the provided text.
528
- 3. After recording, click the "Save" button.
529
-
530
- ### Privacy Policy:
531
-
532
- - Your voice recordings and metadata will be used for public research purposes.
533
- - Please do not share personally identifiable information.
534
- - This dataset will be open-source and published on Hugging Face.
535
- """)
536
-
537
- # Initialize the count
538
- app.load(fn=update_count, inputs=None, outputs=count_display)
539
-
540
  return app
541
 
542
- # Launch the app
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
543
  if __name__ == "__main__":
544
- app = build_ui()
545
- app.launch()
 
 
 
 
6
  import datetime
7
  import uuid
8
  import json
9
+ from huggingface_hub import HfApi, create_repo
10
  from datasets import Dataset
11
+ import soundfile as sf # Added for explicit use in save_recording
12
+ import shutil # Added for explicit use in save_recording
13
 
14
  # Configuration
15
  SAMPLE_PROMPTS = [
 
40
  "सुदूरपश्चिम प्रदेश (Sudurpashchim Province)"
41
  ]
42
 
 
43
  COMMON_LAST_NAMES = {
44
  "पहाडी (Pahadi)": ["शर्मा (Sharma)", "पौडेल (Poudel)", "खनाल (Khanal)", "अधिकारी (Adhikari)", "भट्टराई (Bhattarai)", "अन्य पहाडी (Other Pahadi)"],
45
  "नेवार (Newar)": ["श्रेष्ठ (Shrestha)", "प्रधान (Pradhan)", "महर्जन (Maharjan)", "बज्राचार्य (Bajracharya)", "अन्य नेवार (Other Newar)"],
 
54
  "अन्य (Other)": ["अन्य (Other)"]
55
  }
56
 
57
+ # --- Directory and File Paths ---
58
+ # These paths are relative to where app.py is run.
59
+ # In a Hugging Face Space, this means they are within the Space's file system.
60
+ RECORDINGS_DIR = "recordings"
61
+ METADATA_DIR = "metadata"
62
+ RATINGS_DIR = "ratings"
63
+ METADATA_FILE = os.path.join(METADATA_DIR, "metadata.csv")
64
+ RATINGS_FILE = os.path.join(RATINGS_DIR, "ratings.json")
65
 
66
+ # --- Initialization ---
67
+ def initialize_data_storage():
68
+ """Creates directories and initial files if they don't exist."""
69
+ os.makedirs(RECORDINGS_DIR, exist_ok=True)
70
+ os.makedirs(METADATA_DIR, exist_ok=True)
71
+ os.makedirs(RATINGS_DIR, exist_ok=True)
72
 
73
+ if not os.path.exists(METADATA_FILE):
74
+ pd.DataFrame(columns=[
75
+ "id", "text", "audio_path", "gender", "age_group", "ethnicity",
76
+ "last_name", "region", "emotion", "timestamp", "recording_type"
77
+ ]).to_csv(METADATA_FILE, index=False)
78
 
79
+ if not os.path.exists(RATINGS_FILE):
80
+ with open(RATINGS_FILE, 'w') as f:
81
+ json.dump({}, f)
82
 
83
+ initialize_data_storage() # Call initialization at script start
84
+
85
+ # --- Core Functions ---
86
  def save_recording(audio, text, gender, age_group, ethnicity, last_name, region, emotion, recording_type):
87
  """Save the recording and metadata"""
 
 
 
 
 
88
  if audio is None:
89
  return "कृपया पहिले रेकर्डिङ गर्नुहोस्। (Please record audio first)", None
90
+
91
+ recording_id = str(uuid.uuid4())
92
+ timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
93
+ audio_filename_relative = f"{recording_id}.wav"
94
+ audio_filepath_in_space = os.path.join(RECORDINGS_DIR, audio_filename_relative)
95
+
96
+ try:
97
+ if isinstance(audio, tuple): # If it's a tuple (sr, data) from gr.Audio(type="numpy")
98
+ sr, data = audio
99
+ sf.write(audio_filepath_in_space, data, sr)
100
+ elif isinstance(audio, str) and os.path.exists(audio): # If it's a path from gr.Audio(type="filepath")
101
+ shutil.copy(audio, audio_filepath_in_space)
102
+ # Gradio might place temp files elsewhere, so we ensure it's in our recordings dir
103
+ else:
104
+ return "अडियो फाइल बचत गर्न सकिएन। (Could not save audio file. Invalid audio format.)", None
105
+ except Exception as e:
106
+ return f"अडियो फाइल बचत गर्दा त्रुटि भयो: {e} (Error saving audio file: {e})", None
107
+
108
+
109
+ metadata_df = pd.read_csv(METADATA_FILE)
110
  new_row = pd.DataFrame([{
111
  "id": recording_id,
112
  "text": text,
113
+ "audio_path": audio_filepath_in_space, # Store path relative to space root
114
  "gender": gender,
115
  "age_group": age_group,
116
  "ethnicity": ethnicity,
 
120
  "timestamp": timestamp,
121
  "recording_type": recording_type
122
  }])
123
+
124
+ updated_metadata = pd.concat([metadata_df, new_row], ignore_index=True)
125
+ updated_metadata.to_csv(METADATA_FILE, index=False)
126
+
127
+ with open(RATINGS_FILE, 'r+') as f:
 
128
  ratings = json.load(f)
129
+ ratings[recording_id] = {
130
+ "upvotes": 0, "downvotes": 0,
131
+ "quality_score": 0, "quality_votes": 0,
132
+ "correctness_score": 0, "correctness_votes": 0
133
+ }
134
+ f.seek(0)
 
 
 
 
 
135
  json.dump(ratings, f, indent=2)
136
+ f.truncate()
137
+
138
+ return f"रेकर्डिङ सफलतापूर्वक सुरक्षित गरियो! ID: {recording_id} (Recording saved successfully!)", None # Return None to clear audio input
139
 
140
  def get_random_prompt():
 
141
  return random.choice(SAMPLE_PROMPTS)
142
 
143
+ def get_ethnicity_based_last_names(ethnicity):
144
+ return gr.Dropdown.update(choices=COMMON_LAST_NAMES.get(ethnicity, COMMON_LAST_NAMES["अन्य (Other)"]))
145
+
146
+ def vote_recording(recording_id, vote_type, vote_value_str): # vote_value comes as string from slider
147
+ if not recording_id:
148
+ return "कृपया पहिले समीक्षा गर्न रेकर्डिङ चयन गर्नुहोस्। (Please select a recording to review first.)"
149
+ if not os.path.exists(RATINGS_FILE):
150
  return "रेटिङ फाइल भेटिएन। (Rating file not found.)"
151
+
152
  try:
153
+ vote_value = int(vote_value_str) # Convert to int for quality/correctness
154
+ except ValueError:
155
+ if vote_type in ["quality", "correctness"]:
156
+ return "अमान्य मत मान। (Invalid vote value.)"
157
+ vote_value = 0 # For upvote/downvote
158
+
159
+ try:
160
+ with open(RATINGS_FILE, 'r+') as f:
161
  ratings = json.load(f)
162
+ if recording_id not in ratings:
163
+ return "रेकर्डिङ आईडी भेटिएन। (Recording ID not found.)"
164
+
165
+ rec_ratings = ratings[recording_id]
166
+ if vote_type == "upvote":
167
+ rec_ratings["upvotes"] += 1
168
+ elif vote_type == "downvote":
169
+ rec_ratings["downvotes"] += 1
170
+ elif vote_type == "quality":
171
+ current_score = rec_ratings["quality_score"]
172
+ current_votes = rec_ratings["quality_votes"]
173
+ new_votes = current_votes + 1
174
+ new_score = ((current_score * current_votes) + vote_value) / new_votes
175
+ rec_ratings["quality_score"] = new_score
176
+ rec_ratings["quality_votes"] = new_votes
177
+ elif vote_type == "correctness":
178
+ current_score = rec_ratings["correctness_score"]
179
+ current_votes = rec_ratings["correctness_votes"]
180
+ new_votes = current_votes + 1
181
+ new_score = ((current_score * current_votes) + vote_value) / new_votes
182
+ rec_ratings["correctness_score"] = new_score
183
+ rec_ratings["correctness_votes"] = new_votes
184
+ else:
185
+ return "अमान्य मतदान प्रकार। (Invalid vote type.)"
186
+
187
+ f.seek(0)
 
 
188
  json.dump(ratings, f, indent=2)
189
+ f.truncate()
190
+ return "मतदान सफलतापूर्वक दर्ता गरियो! (Vote registered successfully!)"
 
191
  except Exception as e:
192
+ return f"मतदान दर्ता गर्दा त्रुटि: {str(e)} (Error registering vote: {str(e)})"
 
 
 
 
 
 
193
 
194
+ def upload_to_huggingface(dataset_name, admin_password_attempt):
195
  """Upload the collected data to Hugging Face"""
196
+ # --- Admin Password Check ---
197
+ expected_admin_password = os.environ.get("ADMIN_UPLOAD_PASSWORD")
198
+ hf_token_from_secret = os.environ.get("HF_TOKEN")
199
+
200
+ if not expected_admin_password:
201
+ return "त्रुटि: प्रशासक पासवर्ड स्पेस गोप्यमा कन्फिगर गरिएको छैन। (Error: Admin password not configured in Space secrets.)"
202
+ if admin_password_attempt != expected_admin_password:
203
+ return "त्रुटि: अपलोडका लागि अमान्य प्रशासक पासवर्ड। (Error: Invalid admin password for upload.)"
204
+ if not hf_token_from_secret:
205
+ return "त्रुटि: HF_TOKEN गोप्य स्पेस कन्फिगरेसनमा फेला परेन। अपलोड गर्न सकिँदैन। (Error: HF_TOKEN secret not found in Space configuration. Cannot upload.)"
206
+ if not dataset_name or len(dataset_name.split('/')) != 2:
207
+ return "त्रुटि: कृपया मान्य डेटासेट नाम 'username/repo_name' ढाँचामा प्रदान गर्नुहोस्। (Error: Please provide a valid dataset name in 'username/repo_name' format.)"
208
+
209
+ if not os.path.exists(METADATA_FILE):
210
+ return "कुनै मेटाडाटा फाइल भेटिएन। (No metadata file found.)"
211
+
212
  try:
213
+ api = HfApi(token=hf_token_from_secret)
214
+ # Ensure repo exists, create if not. private=False for public dataset
215
+ create_repo(repo_id=dataset_name, token=hf_token_from_secret, repo_type="dataset", exist_ok=True, private=False)
216
+
217
+ metadata = pd.read_csv(METADATA_FILE)
218
  if len(metadata) == 0:
219
+ return "कुनै डाटा भेटिएन। (No data to upload.)"
220
+
221
+ with open(RATINGS_FILE, 'r') as f:
222
+ ratings_data = json.load(f)
223
+
224
+ metadata["upvotes"] = metadata["id"].apply(lambda x: ratings_data.get(x, {}).get("upvotes", 0))
225
+ metadata["downvotes"] = metadata["id"].apply(lambda x: ratings_data.get(x, {}).get("downvotes", 0))
226
+ metadata["quality_score"] = metadata["id"].apply(lambda x: ratings_data.get(x, {}).get("quality_score", 0))
227
+ metadata["quality_votes"] = metadata["id"].apply(lambda x: ratings_data.get(x, {}).get("quality_votes", 0))
228
+ metadata["correctness_score"] = metadata["id"].apply(lambda x: ratings_data.get(x, {}).get("correctness_score", 0))
229
+ metadata["correctness_votes"] = metadata["id"].apply(lambda x: ratings_data.get(x, {}).get("correctness_votes", 0))
230
+
231
+ # Prepare audio column for datasets library
232
+ # The 'audio' column should contain dictionaries with 'path' and optionally 'bytes'
233
+ # Here, we'll tell datasets to load from the paths we upload.
234
+ audio_files_for_dataset = []
235
+ for audio_path_in_space in metadata["audio_path"]:
236
+ audio_files_for_dataset.append(
237
+ {"path": os.path.join("audio", os.path.basename(audio_path_in_space))}
238
+ )
239
 
240
+ dataset_dict = metadata.to_dict(orient='list')
241
+ dataset_dict['audio'] = audio_files_for_dataset # Add the audio column
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
242
 
243
+ # Remove the local audio_path column as we now have the 'audio' dict column
244
+ if 'audio_path' in dataset_dict:
245
+ del dataset_dict['audio_path']
246
+
247
+ hf_dataset = Dataset.from_dict(dataset_dict)
248
 
249
+ # Push dataset metadata (e.g., data.jsonl or data.arrow/parquet files in the repo)
250
+ hf_dataset.push_to_hub(repo_id=dataset_name) # token is implicitly used if HfApi was init with it or HF_TOKEN env var is set
251
+
252
+ # Upload individual audio files
253
+ # Create the audio folder in the dataset repo if it doesn't exist
254
+ try:
255
+ api.create_folder(
256
+ repo_id=dataset_name,
257
+ folder_path="audio", # Target folder in the dataset repo
258
+ repo_type="dataset",
259
+ exist_ok=True
260
+ )
261
+ except Exception as e:
262
+ # Log this, but it's not critical if the folder already exists
263
+ print(f"सूचना: अडियो फोल्डर सिर्जना गर्न सकिएन (यो पहिले नै अवस्थित हुन सक्छ): {e} (Info: Could not create audio folder (it might already exist): {e})")
264
 
265
+ upload_count = 0
266
  for _, row in metadata.iterrows():
267
+ local_audio_file = row["audio_path"] # This is like "recordings/uuid.wav"
268
+ if os.path.exists(local_audio_file):
269
+ # The path_in_repo should match what you put in the 'audio' column for datasets
270
+ target_path_in_repo = os.path.join("audio", os.path.basename(local_audio_file))
271
  api.upload_file(
272
+ path_or_fileobj=local_audio_file,
273
+ path_in_repo=target_path_in_repo,
274
  repo_id=dataset_name,
275
  repo_type="dataset"
276
  )
277
+ upload_count +=1
278
+
279
+ return (f"डाटा हगिङफेसमा सफलतापूर्वक अपलोड गरियो! {upload_count} अडियो फाइलहरू अपलोड गरियो। "
280
+ f"(Data successfully uploaded to Hugging Face at {dataset_name}. {upload_count} audio files uploaded.)")
281
+
282
  except Exception as e:
283
+ import traceback
284
+ tb_str = traceback.format_exc()
285
+ return f"अपलोडको क्रममा त्रुटि (Error during upload):\n{str(e)}\n{tb_str}"
286
 
287
  def update_count():
288
+ if os.path.exists(METADATA_FILE):
289
+ try:
290
+ metadata = pd.read_csv(METADATA_FILE)
291
+ return f"हालसम्म {len(metadata)} रेकर्डिङहरू संकलन गरिएको छ। (Total recordings collected: {len(metadata)})"
292
+ except pd.errors.EmptyDataError:
293
+ return "हालसम्म ० रेकर्डिङहरू संकलन गरिएको छ। (Total recordings collected: 0)"
294
  return "कुनै रेकर्डिङ भेटिएन। (No recordings found.)"
295
 
296
  def list_recordings(num_items=10):
297
+ if not os.path.exists(METADATA_FILE):
298
+ return pd.DataFrame(columns=['id', 'text', 'ethnicity', 'region', 'timestamp'])
299
+ try:
300
+ metadata = pd.read_csv(METADATA_FILE)
301
+ except pd.errors.EmptyDataError:
302
+ return pd.DataFrame(columns=['id', 'text', 'ethnicity', 'region', 'timestamp'])
303
+
304
  if len(metadata) == 0:
305
+ return pd.DataFrame(columns=['id', 'text', 'ethnicity', 'region', 'timestamp'])
306
+
307
+ metadata['timestamp'] = pd.to_datetime(metadata['timestamp'], errors='coerce')
308
+ sorted_metadata = metadata.sort_values('timestamp', ascending=False).head(int(num_items))
 
 
 
309
  display_df = sorted_metadata[['id', 'text', 'ethnicity', 'region', 'timestamp']].copy()
310
+ display_df['timestamp'] = display_df['timestamp'].dt.strftime('%Y-%m-%d %H:%M').fillna('N/A')
311
+ return display_df.reset_index(drop=True)
 
 
312
 
313
  def get_recording_audio(recording_id):
314
+ if not recording_id: return None, "कुनै रेकर्डिङ आईडी प्रदान गरिएको छैन। (No recording ID provided.)"
315
+ if not os.path.exists(METADATA_FILE): return None, "मेटाडाटा फाइल भेटिएन। (Metadata file not found.)"
316
+ try:
317
+ metadata = pd.read_csv(METADATA_FILE)
318
+ except pd.errors.EmptyDataError:
319
+ return None, "मेटाडाटा खाली छ। (Metadata is empty.)"
320
+
321
  recording = metadata[metadata['id'] == recording_id]
322
+ if len(recording) == 0: return None, "रेकर्डिङ भेटिएन। (Recording not found.)"
323
+
 
 
324
  audio_path = recording['audio_path'].iloc[0]
325
  text = recording['text'].iloc[0]
326
+ if not os.path.exists(audio_path): return None, f"अडियो फाइल भेटिएन: {audio_path} (Audio file not found: {audio_path})"
 
 
 
327
  return audio_path, text
328
 
329
  def get_recording_ratings(recording_id):
330
+ if not recording_id: return "रेकर्डिङ आईडी चयन गर्नुहोस्। (Select a Recording ID.)"
331
+ if not os.path.exists(RATINGS_FILE): return "डाटा भेटिएन। (No ratings data found.)"
332
+
333
+ with open(RATINGS_FILE, 'r') as f:
 
334
  ratings = json.load(f)
335
+ if recording_id not in ratings: return "यस रेकर्डिङको लागि कुनै मूल्याङ्कन भेटिएन। (No ratings found for this recording.)"
336
+
 
 
337
  r = ratings[recording_id]
338
+ upvotes = r.get("upvotes", 0)
339
+ downvotes = r.get("downvotes", 0)
340
+ quality = round(r.get("quality_score",0), 1) if r.get("quality_votes",0) > 0 else 0
341
+ quality_votes = r.get("quality_votes",0)
342
+ correctness = round(r.get("correctness_score",0), 1) if r.get("correctness_votes",0) > 0 else 0
343
+ correctness_votes = r.get("correctness_votes",0)
344
+
 
 
345
  return f"""👍 Upvotes: {upvotes} | 👎 Downvotes: {downvotes}
346
  गुणस्तर (Quality): {quality}/5 ({quality_votes} मत/votes)
347
  शुद्धता (Correctness): {correctness}/5 ({correctness_votes} मत/votes)"""
348
 
349
+ # --- Gradio UI Build ---
350
  def build_ui():
 
351
  with gr.Blocks(title="नेपाली ASR डाटा संकलन (Nepali ASR Data Collection)") as app:
352
+ gr.Markdown("# नेपाली ASR डाटा संकलन (Nepali ASR Data Collection)")
353
+ gr.Markdown(
354
+ "यस प्लेटफर्मले नेपाली भाषाको स्वचालित भाषण पहिचान (ASR) प्रविधिको विकासका लागि आवाज डाटा संकलन गर्दछ। "
355
+ "कृपया आफ्नो आवाज रेकर्ड गरेर योगदान दिनुहोस्।\n"
356
+ "*This platform collects voice data for the development of Nepali Automatic Speech Recognition (ASR) technology. "
357
+ "Please contribute by recording your voice.*"
358
+ )
359
+
360
+ # --- Data Collection Tabs ---
361
+ with gr.Tabs():
362
+ with gr.TabItem("१. आवाज रेकर्ड गर्नुहोस् (Record Voice)"):
363
+ with gr.Tabs():
364
+ with gr.TabItem("स्वतन्त्र पाठ (Free Text)"):
365
+ with gr.Row():
366
+ with gr.Column(scale=2):
367
+ free_text_input = gr.Textbox(label="तपाईंले बोल्न चाहनुभएको पाठ (Text you want to speak)", placeholder="यहाँ लेख्नुहोस्...", lines=3)
368
+ free_audio_input = gr.Audio(label="आवाज रेकर्ड गर्नुहोस् (Record your voice)", type="filepath", source="microphone")
369
+ with gr.Column(scale=3):
370
+ with gr.Row():
371
+ free_gender_dd = gr.Dropdown(label="लिङ्ग (Gender)", choices=GENDERS, value=GENDERS[0])
372
+ free_age_dd = gr.Dropdown(label="उमेर समूह (Age Group)", choices=AGE_GROUPS, value=AGE_GROUPS[1])
373
+ with gr.Row():
374
+ free_ethnicity_dd = gr.Dropdown(label="जातीयता (Ethnicity)", choices=list(COMMON_LAST_NAMES.keys()), value=list(COMMON_LAST_NAMES.keys())[0])
375
+ free_lastname_dd = gr.Dropdown(label="थर (Last Name)", choices=COMMON_LAST_NAMES[list(COMMON_LAST_NAMES.keys())[0]])
376
+ free_ethnicity_dd.change(fn=get_ethnicity_based_last_names, inputs=free_ethnicity_dd, outputs=free_lastname_dd)
377
+ with gr.Row():
378
+ free_region_dd = gr.Dropdown(label="क्षेत्र (Region)", choices=REGIONS, value=REGIONS[2])
379
+ free_emotion_dd = gr.Dropdown(label="भावना (Emotion)", choices=EMOTIONS, value=EMOTIONS[0])
380
+ free_submit_btn = gr.Button("सुरक्षित गर्नुहोस् (Save Free Text Recording)")
381
+ free_status_output = gr.Textbox(label="स्थिति (Status)", interactive=False)
382
+ free_submit_btn.click(
383
+ save_recording,
384
+ inputs=[free_audio_input, free_text_input, free_gender_dd, free_age_dd, free_ethnicity_dd, free_lastname_dd, free_region_dd, free_emotion_dd, gr.Textbox(value="free_text", visible=False)],
385
+ outputs=[free_status_output, free_audio_input] # Clear audio on success
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
386
  )
387
+
388
+ with gr.TabItem("निर्देशित पाठ (Prompted Text)"):
389
+ with gr.Row():
390
+ with gr.Column(scale=2):
391
+ prompt_text_display = gr.Textbox(label="कृपया यो पाठ पढ्नुहोस् (Please read this text)", value=get_random_prompt(), lines=3, interactive=False)
392
+ new_prompt_btn = gr.Button("नयाँ पाठ (New Prompt)")
393
+ prompt_audio_input = gr.Audio(label="आवाज रेकर्ड गर्नुहोस् (Record your voice)", type="filepath", source="microphone")
394
+ with gr.Column(scale=3):
395
+ with gr.Row():
396
+ prompt_gender_dd = gr.Dropdown(label="लिङ्ग (Gender)", choices=GENDERS, value=GENDERS[0])
397
+ prompt_age_dd = gr.Dropdown(label="उमेर समूह (Age Group)", choices=AGE_GROUPS, value=AGE_GROUPS[1])
398
+ with gr.Row():
399
+ prompt_ethnicity_dd = gr.Dropdown(label="जातीयता (Ethnicity)", choices=list(COMMON_LAST_NAMES.keys()), value=list(COMMON_LAST_NAMES.keys())[0])
400
+ prompt_lastname_dd = gr.Dropdown(label="थर (Last Name)", choices=COMMON_LAST_NAMES[list(COMMON_LAST_NAMES.keys())[0]])
401
+ prompt_ethnicity_dd.change(fn=get_ethnicity_based_last_names, inputs=prompt_ethnicity_dd, outputs=prompt_lastname_dd)
402
+ with gr.Row():
403
+ prompt_region_dd = gr.Dropdown(label="क्षेत्र (Region)", choices=REGIONS, value=REGIONS[2])
404
+ prompt_emotion_dd = gr.Dropdown(label="भावना (Emotion)", choices=EMOTIONS, value=EMOTIONS[0])
405
+ new_prompt_btn.click(get_random_prompt, outputs=prompt_text_display)
406
+ prompt_submit_btn = gr.Button("सुरक्षित गर्नुहोस् (Save Prompted Recording)")
407
+ prompt_status_output = gr.Textbox(label="स्थिति (Status)", interactive=False)
408
+ prompt_submit_btn.click(
409
+ save_recording,
410
+ inputs=[prompt_audio_input, prompt_text_display, prompt_gender_dd, prompt_age_dd, prompt_ethnicity_dd, prompt_lastname_dd, prompt_region_dd, prompt_emotion_dd, gr.Textbox(value="prompted_text", visible=False)],
411
+ outputs=[prompt_status_output, prompt_audio_input]
412
  )
413
+
414
+ with gr.TabItem("२. रेकर्डिङ समीक्षा गर्नुहोस् (Review Recordings)"):
415
+ gr.Markdown("हालसालैका रेकर्डिङहरू हेर्नुहोस् र मत दिनुहोस्। (View and vote on recent recordings.)")
416
+ num_review_items = gr.Number(value=10, label="देखाउने वस्तुहरूको संख्या (Number of items to show)", minimum=1, maximum=50, step=1)
417
+ refresh_review_list_btn = gr.Button("सूची ताजा गर्नुहोस् (Refresh List)")
418
+ review_list_df = gr.DataFrame(headers=['id', 'text', 'ethnicity', 'region', 'timestamp'], label="हालका रेकर्डिङहरू (Recent Recordings)", interactive=False, datatype=['str', 'str', 'str', 'str', 'str'])
419
+
420
+ with gr.Row():
421
+ selected_review_id = gr.Textbox(label="चयन गरिएको आईडी (Selected ID)", interactive=False)
422
+ selected_review_text = gr.Textbox(label="रेकर्डिङ पाठ (Recording Text)", interactive=False, lines=2)
423
+ review_audio_player = gr.Audio(label="रेकर्डिङ सुन्नुहोस् (Listen to Recording)", type="filepath")
424
+ current_ratings_display = gr.Textbox(label="वर्तमान मूल्याङ्कन (Current Ratings)", interactive=False, lines=3)
425
+
426
+ def select_for_review(evt: gr.SelectData, df_data: pd.DataFrame):
427
+ if evt.index is None or df_data is None or len(df_data) == 0 or evt.index[0] >= len(df_data):
428
+ return "", "", None, "कुनै रेकर्डिङ चयन गरिएको छैन (No recording selected)"
429
+ selected_id_val = df_data.iloc[evt.index[0]]['id']
430
+ audio_p, text_val = get_recording_audio(selected_id_val)
431
+ ratings_text_val = get_recording_ratings(selected_id_val)
432
+ return selected_id_val, text_val, audio_p, ratings_text_val
433
+
434
+ review_list_df.select(select_for_review, inputs=[review_list_df], outputs=[selected_review_id, selected_review_text, review_audio_player, current_ratings_display])
435
+ refresh_review_list_btn.click(list_recordings, inputs=[num_review_items], outputs=review_list_df)
436
+
437
+ gr.Markdown("### मतदान गर्नुहोस् (Cast Your Vote)")
438
+ with gr.Row():
439
+ upvote_btn = gr.Button("👍 मन पर्यो (Upvote)")
440
+ downvote_btn = gr.Button("👎 मन परेन (Downvote)")
441
+ with gr.Row():
442
+ quality_rating_slider = gr.Slider(minimum=1, maximum=5, step=1, label="गुणस्तर मूल्याङ्कन (Quality Rating 1-5)", value=3)
443
+ submit_quality_btn = gr.Button("गुणस्तर मत दिनुहोस् (Submit Quality)")
444
+ with gr.Row():
445
+ correctness_rating_slider = gr.Slider(minimum=1, maximum=5, step=1, label="शुद्धता मूल्याङ्कन (Correctness Rating 1-5)", value=3)
446
+ submit_correctness_btn = gr.Button("शुद्धता मत दिनुहोस् (Submit Correctness)")
447
+ vote_status_output = gr.Textbox(label="मतदान स्थिति (Voting Status)", interactive=False)
448
+
449
+ def vote_and_refresh(rec_id, vote_t, vote_val_str):
450
+ status = vote_recording(rec_id, vote_t, str(vote_val_str)) # Ensure vote_val is str
451
+ new_ratings = get_recording_ratings(rec_id) if rec_id else "रेकर्डिङ चयन गर्नुहोस् (Select a recording)"
452
+ # Also refresh the main list to reflect potential score changes indirectly
453
+ # latest_list = list_recordings(num_review_items.value) # This needs to be handled carefully to avoid component errors
454
+ return status, new_ratings
455
+
456
+ upvote_btn.click(vote_and_refresh, inputs=[selected_review_id, gr.Textbox(value="upvote", visible=False), gr.Number(value=0, visible=False)], outputs=[vote_status_output, current_ratings_display])
457
+ downvote_btn.click(vote_and_refresh, inputs=[selected_review_id, gr.Textbox(value="downvote", visible=False), gr.Number(value=0, visible=False)], outputs=[vote_status_output, current_ratings_display])
458
+ submit_quality_btn.click(vote_and_refresh, inputs=[selected_review_id, gr.Textbox(value="quality", visible=False), quality_rating_slider], outputs=[vote_status_output, current_ratings_display])
459
+ submit_correctness_btn.click(vote_and_refresh, inputs=[selected_review_id, gr.Textbox(value="correctness", visible=False), correctness_rating_slider], outputs=[vote_status_output, current_ratings_display])
460
+
461
+ with gr.TabItem("३. प्रगति र अपलोड (Progress & Upload)"):
462
+ gr.Markdown("## संकलन प्रगति (Collection Progress)")
463
+ total_count_display = gr.Textbox(label="कुल संकलित रेकर्डिङ (Total Recordings Collected)", interactive=False)
464
+ refresh_total_count_btn = gr.Button("गणना ताजा गर्नुहोस् (Refresh Count)")
465
+ refresh_total_count_btn.click(update_count, outputs=total_count_display)
466
+
467
+ gr.Markdown("---")
468
+ gr.Markdown("## हगिङफेसमा अपलोड गर्नुहोस् (Upload to Hugging Face)")
469
+ gr.Markdown(
470
+ "**महत्वपूर्ण:** यो कार्यले स्पेसमा संकलित सबै डाटालाई हगिङ फेस डेटासेटमा पुश गर्नेछ। "
471
+ "स्पेसको स्टोरेज अस्थायी हुन सक्छ, त्यसैले नियमित रूपमा अपलोड गर्न सिफारिस गरिन्छ।\n"
472
+ "यो कार्य गर्नको लागि, तपाईंले स्पेस सेटिङहरूमा `HF_TOKEN` (लेख्ने पहुँच सहितको हगिङ फेस टोकन) "
473
+ "र `ADMIN_UPLOAD_PASSWORD` गोप्य रूपमा थप्नुपर्छ।\n\n"
474
+ "**IMPORTANT:** This action will push all data collected in this Space to the Hugging Face Dataset. "
475
+ "Space storage can be ephemeral, so regular uploads are recommended. "
476
+ "To perform this action, you must have added `HF_TOKEN` (a Hugging Face token with write access) "
477
+ "and `ADMIN_UPLOAD_PASSWORD` as secrets in the Space settings."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
478
  )
479
+ hf_dataset_name_input = gr.Textbox(label="Dataset Name (e.g., your_username/nepali-asr-data)", placeholder="your_hf_username/dataset_repo_name")
480
+ admin_password_input = gr.Textbox(label="Admin Upload Password", type="password", placeholder="Enter admin password")
481
+ upload_to_hf_btn = gr.Button("हगिङफेसमा अपलोड गर्नुहोस् (Upload to Hugging Face)")
482
+ upload_status_output = gr.Textbox(label="अपलोड स्थिति (Upload Status)", interactive=False, lines=5)
483
+ upload_to_hf_btn.click(upload_to_huggingface, inputs=[hf_dataset_name_input, admin_password_input], outputs=upload_status_output)
484
+
485
+ with gr.TabItem("४. जानकारी (Information)"):
486
+ gr.Markdown(render_info_page()) # Using a helper for cleaner code
487
+
488
+ # Initial loads
489
+ app.load(fn=update_count, inputs=None, outputs=total_count_display)
490
+ app.load(fn=lambda n: list_recordings(n), inputs=[num_review_items], outputs=review_list_df) # Load initial review list
491
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
492
  return app
493
 
494
+ def render_info_page():
495
+ return """
496
+ ## नेपाली ASR डाटा संकलन प्रोजेक्टको बारेमा (About the Nepali ASR Data Collection Project)
497
+
498
+ यो प्रोजेक्टले नेपाली भाषाको स्वचालित भाषण पहिचान (ASR) प्रविधिको विकासका लागि आवश्यक डाटा संकलन गर्दछ।
499
+ तपाईंको योगदानले नेपाली भाषा प्रविधिको विकासमा ठूलो मद्दत पुर्‍याउनेछ।
500
+
501
+ ### कसरी योगदान दिने (How to Contribute):
502
+ 1. **आवाज रेकर्ड गर्नुहोस् (Record Voice)** ट्याबमा जानुहोस्।
503
+ * **स्वतन्त्र पाठ (Free Text)** अन्तर्गत, तपाईं आफ्नो इच्छा अनुसारको पाठ लेख्नुहोस्, आवश्यक विवरणहरू (लिङ्ग, उमेर, आदि) छान्नुहोस्, र आफ्नो आवाज रेकर्ड गर्नुहोस्।
504
+ * **निर्देशित पाठ (Prompted Text)** अन्तर्गत, दिइएको नेपाली वाक्य पढ्नुहोस्, विवरणहरू छान्नुहोस्, र आफ्नो आवाज रेकर्ड गर्नुहोस्। "नयाँ पाठ" बटनले तपाईंलाई फरक वाक्य दिनेछ।
505
+ 2. **रेकर्डिङ समीक्षा गर्नुहोस् (Review Recordings)** ट्याबमा गएर अरूले गरेका रेकर्डिङहरू सुन्नुहोस् र तिनीहरूको गुणस्तर र शुद्धताको लागि मतदान गर्नुहोस्। यसले डाटाको गुणस्तर सुधार गर्न मद्दत गर्दछ।
506
+ 3. रेकर्डिङ पछि, "सुरक्षित गर्नुहोस् (Save)" बटनमा क्लिक गर्नुहोस्।
507
+
508
+ ### गोपनीयता नीति (Privacy Policy):
509
+ - तपाईंको आवाज रेकर्डिङ र सम्बन्धित मेटाडाटा (जस्तै उमेर समूह, लिङ्ग, क्षेत्र) सार्वजनिक अनुसन्धान उद्देश्यका लागि प्रयोग गरिनेछ।
510
+ - हामी तपाईंको नाम वा सम्पर्क जानकारी जस्ता प्रत्यक्ष व्यक्तिगत पहिचान योग्य जानकारी सङ्कलन गर्दैनौं। तपाईंले प्रदान गर्नुभएको जातीयता/थरको जानकारी उच्चारण र विविधता अध्ययनको लागि हो।
511
+ - यो डाटासेट खुला स्रोत हुनेछ र हगिङ फेस जस्ता प्लेटफर्महरूमा अनुसन्धान समुदायको लागि उपलब्ध गराइनेछ।
512
+ - कृपया रेकर्डिङको क्रममा कुनै पनि संवेदनशील व्यक्तिगत जानकारी नबोल्नुहोस्।
513
+
514
+ ---
515
+
516
+ ## About Nepali ASR Data Collection Project
517
+
518
+ This project collects voice data essential for developing Automatic Speech Recognition (ASR) technology for the Nepali language.
519
+ Your contribution will significantly aid in the advancement of Nepali language technology.
520
+
521
+ ### How to Contribute:
522
+ 1. Go to the **Record Voice (आवाज रेकर्ड गर्नुहोस्)** tab.
523
+ * Under **Free Text (स्वतन्त्र पाठ)**, type any Nepali text you wish, select the required demographic details (gender, age, etc.), and record your voice.
524
+ * Under **Prompted Text (निर्देशित पाठ)**, read the provided Nepali sentence, select demographic details, and record your voice. The "New Text (नयाँ पाठ)" button will give you a different sentence.
525
+ 2. Go to the **Review Recordings (रेकर्डिङ समीक्षा गर्नुहोस्)** tab to listen to recordings made by others and vote on their quality and correctness. This helps improve the overall quality of the dataset.
526
+ 3. After recording, click the "Save (सुरक्षित गर्नुहोस्)" button.
527
+
528
+ ### Privacy Policy:
529
+ - Your voice recordings and associated metadata (like age group, gender, region) will be used for public research purposes.
530
+ - We do not collect directly personally identifiable information such as your name or contact details. The ethnicity/last name information you provide is for studying accent and diversity.
531
+ - This dataset will be open-source and made available to the research community on platforms like Hugging Face.
532
+ - Please do not speak any sensitive personal information during your recordings.
533
+ """
534
+
535
+ # --- Main Execution ---
536
  if __name__ == "__main__":
537
+ # Ensure storage is initialized when running locally too
538
+ initialize_data_storage()
539
+
540
+ app_ui = build_ui()
541
+ app_ui.launch()