cleaning and add recording verification and cuting samples in 2 seconds

app.py

@@ -7,6 +7,10 @@ import tempfile
 from datetime import datetime
 from huggingface_hub import HfApi, hf_hub_download,CommitOperationDelete
 from pathlib import Path
+import librosa
+import soundfile as sf
+import tempfile
+import numpy as np
 
 
 label_codes = {
@@ -19,7 +23,7 @@ label_decoder = {v: k for k, v in label_codes.items()}
 # --- CONFIGURATION ---
 DATASET_REPO_ID = "MeysamSh/ENSIMSoundDataCollection"
 HF_TOKEN = os.environ.get("HF_TOKEN")
-COUPON_SALT = os.environ.get("COUPON_SALT")#"ENSIM_2026_SECRET"
+COUPON_SALT = os.environ.get("COUPON_SALT")
 
 # Admin Credentials
 ADMIN_USERNAME = "admin"
@@ -47,32 +51,92 @@ def verify_user(email):
         return gr.update(visible=True), f"✅ Access Granted: {clean_email}"
     return gr.update(visible=False), "🚫 Not authorized."
 
+
 def upload_data(email, label, audio_path):
-    if not audio_path: return "⚠️ No audio.", ""
+    # --- Energy Threshold Setting ---
+    ENERGY_THRESHOLD = 0.02 # Adjust this: 0.01 is very sensitive, 0.05 is strict
+    
+    if audio_path is None:
+        return "⚠️ Please record or upload a sound file.", None, gr.update(), ""
+    if not label:
+        return "⚠️ Please select a category label.", gr.update(), gr.update(), ""
+
     try:
-        clean_email = email.strip().lower()
-        email_index = AUTHORIZED_USERS.index(clean_email) if clean_email in AUTHORIZED_USERS else "unknown"
-        
-        # Determine if this is file #1, #2, etc. for this user to assign Training/Test
-        all_files = api.list_repo_files(repo_id=DATASET_REPO_ID, repo_type="dataset")
-        user_submissions = [f for f in all_files if f.startswith(f"data/{email_index}_")]
-        submission_number = len(user_submissions) + 1
+        y, sr = librosa.load(audio_path, sr=None)
+        duration = librosa.get_duration(y=y, sr=sr)
         
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        unique_filename = f"{email_index}_{timestamp}.wav"
+        if duration < 2.0:
+            return f"⚠️ Sound too short ({duration:.1f}s).", gr.update(), gr.update(), ""
+
+        raw_segments = []
         
-        # Upload Audio
-        api.upload_file(path_or_fileobj=audio_path, path_in_repo=f"data/{unique_filename}",
-                        repo_id=DATASET_REPO_ID, repo_type="dataset", token=HF_TOKEN)
-        coupon = generate_coupon(unique_filename)
-        # Upload Metadata
-        meta_content = f"user_id,label,file_name,time,order\n{clean_email},{label},{unique_filename},{timestamp},{submission_number}"
-        api.upload_file(path_or_fileobj=meta_content.encode(), path_in_repo=f"metadata/meta_{email_index}_{timestamp}.csv",
-                        repo_id=DATASET_REPO_ID, repo_type="dataset", token=HF_TOKEN)
+        # --- SPLITTING LOGIC ---
+        if duration < 5.0:
+            raw_segments.append(y[:int(2 * sr)])
+        elif duration >= 7.0:
+            start_sample = int(3 * sr)
+            remaining_audio = y[start_sample:]
+            window_size = int(2 * sr)
+            for i in range(0, len(remaining_audio) - window_size + 1, window_size):
+                raw_segments.append(remaining_audio[i : i + window_size])
+        else:
+            raw_segments.append(y[:int(2 * sr)])
+
+        # --- ENERGY CALCULATION & FILTERING ---
+        valid_segments = []
+        rejected_count = 0
         
+        for seg in raw_segments:
+            # Calculate RMS energy: sqrt(mean(x^2))
+            rms = np.sqrt(np.mean(seg**2))
+            
+            if rms >= ENERGY_THRESHOLD:
+                valid_segments.append(seg)
+            else:
+                rejected_count += 1
+
+        if not valid_segments:
+            return f"❌ Rejected: {rejected_count} segments were too quiet. Please record closer to the source.", None, gr.update(), ""
+
+        # --- UPLOAD PROCESS ---
+        clean_email = email.strip().lower()
+        email_index = AUTHORIZED_USERS.index(clean_email) if clean_email in AUTHORIZED_USERS else "unknown"
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
         
-        return f"🎉 Success! Submission #{submission_number} saved.", coupon
-    except Exception as e: return f"❌ Error: {str(e)}", ""
+        coupons = []
+        for idx, seg in enumerate(valid_segments):
+            with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp_seg:
+                sf.write(tmp_seg.name, seg, sr)
+                seg_filename = f"{email_index}_{timestamp}_seg{idx}.wav"
+                coupon = generate_coupon(seg_filename)
+                coupons.append(coupon)
+                
+                api.upload_file(
+                    path_or_fileobj=tmp_seg.name,
+                    path_in_repo=f"data/{seg_filename}",
+                    repo_id=DATASET_REPO_ID,
+                    repo_type="dataset",
+                    token=HF_TOKEN
+                )
+                
+                meta_content = f"user_id,label,file_name,time,order\n{clean_email},{label},{seg_filename},{timestamp},{idx+1}"
+                api.upload_file(
+                    path_or_fileobj=meta_content.encode(),
+                    path_in_repo=f"metadata/meta_{email_index}_{timestamp}_seg{idx}.csv",
+                    repo_id=DATASET_REPO_ID,
+                    repo_type="dataset",
+                    token=HF_TOKEN
+                )
+            os.unlink(tmp_seg.name)
+
+        status_msg = f"🎉 Success! {len(valid_segments)} samples accepted."
+        if rejected_count > 0:
+            status_msg += f" ({rejected_count} quiet segments discarded)."
+            
+        return status_msg, None, gr.update(value=None), ", ".join(coupons)
+
+    except Exception as e:
+        return f"❌ Error: {str(e)}", gr.update(), gr.update(), ""
 
 # --- ADMIN LOGIC ---
 
@@ -180,72 +244,72 @@ def delete_selected_file(file_path):
     except Exception as e: return f"❌ Error: {str(e)}", gr.update()
 
 def access_dataset_zip(email, coupons_str):
-    """Verifies coupons and creates a ZIP of accessible data."""
     if not email or not coupons_str:
         return None, "⚠️ Please provide your email and coupons."
     
-    # Clean coupons input
     coupons_list = [c.strip().upper() for c in coupons_str.split(",") if c.strip()]
     num_coupons = len(coupons_list)
     
     if num_coupons == 0:
         return None, "⚠️ No valid coupons provided."
 
-    # try:
-    all_files = api.list_repo_files(repo_id=DATASET_REPO_ID, repo_type="dataset")
-    meta_files = [f for f in all_files if f.startswith("metadata/")]
-    
-    # Create a temporary directory for the ZIP
-    tmp_dir = tempfile.mkdtemp()
-    zip_path = os.path.join(tmp_dir, f"ENSIM_Data_{num_coupons}_coupons.zip")
-    print(f"Creating ZIP at: {zip_path}")
-
-    training_metadata = []
-
-    with zipfile.ZipFile(zip_path, 'w') as zipf:
-        for m_file in meta_files:
-            # Get metadata for each entry
-            print(f"Processing metadata file: {m_file}")
-            local_meta = hf_hub_download(repo_id=DATASET_REPO_ID, filename=m_file, repo_type="dataset", token=HF_TOKEN)
-            print(f"Downloaded metadata to: {local_meta}")
-            df = pd.read_csv(local_meta)
-            row = df.iloc[0]
-            
-            order = int(row['order'])
-            audio_filename = row['file_name']
-            audio_repo_path = f"data/{audio_filename}"
-            print(f"Processing {audio_filename} with order {order} for coupon access...")
-            
-            # ACCESS RULES:
-            # 1. Training Set (Odd): Only if student has enough coupons
-            # 2. Test Set (Even): Always included if they have at least 1 coupon
-            
-            is_training = order % 2 != 0
-            
-            if is_training and order <= num_coupons:
-                # Download and add to Training folder
-                audio_local = hf_hub_download(repo_id=DATASET_REPO_ID, filename=audio_repo_path, repo_type="dataset", token=HF_TOKEN)
-                print(f"Adding to ZIP: {audio_filename} (Label: {row['label']})")
-                zipf.write(audio_local, arcname=f"training_set/{audio_filename}")
-                print(f"Added {audio_filename} to ZIP under training_set/")
-                training_metadata.append({"file": audio_filename, "label": row['label']})
-            
-            elif not is_training:
-                # Download and add to Test folder (No labels added to zip)
-                audio_local = hf_hub_download(repo_id=DATASET_REPO_ID, filename=audio_repo_path, repo_type="dataset", token=HF_TOKEN)
-                print(f"Adding to ZIP: {audio_filename} (Test Set)")
-                zipf.write(audio_local, arcname=f"test_set/{audio_filename}")
-                print(f"Added {audio_filename} to ZIP under test_set/")
-
-        # Add Training Metadata CSV to the ZIP if data exists
-        if training_metadata:
-            print(f"Adding {len(training_metadata)} training metadata entries to ZIP...")
-            train_df = pd.DataFrame(training_metadata)
-            csv_path = os.path.join(tmp_dir, "training_labels.csv")
-            train_df.to_csv(csv_path, index=False)
-            zipf.write(csv_path, arcname="training_set/labels.csv")
-
-    return zip_path, f"✅ Successfully bundled data for {num_coupons} coupons."
+    try:
+        all_files = api.list_repo_files(repo_id=DATASET_REPO_ID, repo_type="dataset")
+        meta_files = [f for f in all_files if f.startswith("metadata/")]
+        
+        tmp_dir = tempfile.mkdtemp()
+        zip_path = os.path.join(tmp_dir, f"ENSIM_Data_Collection.zip")
+        
+        # This list will hold rows for our single combined CSV
+        compiled_metadata = []
+
+        with zipfile.ZipFile(zip_path, 'w') as zipf:
+            for m_file in meta_files:
+                local_meta = hf_hub_download(repo_id=DATASET_REPO_ID, filename=m_file, repo_type="dataset", token=HF_TOKEN)
+                df = pd.read_csv(local_meta)
+                row = df.iloc[0]
+                
+                order = int(row['order'])
+                audio_filename = row['file_name']
+                audio_repo_path = f"data/{audio_filename}"
+                is_training = order % 2 != 0
+                
+                # --- ACCESS LOGIC ---
+                # 1. Training files (Odd): include only if within coupon count
+                if is_training and order <= num_coupons:
+                    audio_local = hf_hub_download(repo_id=DATASET_REPO_ID, filename=audio_repo_path, repo_type="dataset", token=HF_TOKEN)
+                    zipf.write(audio_local, arcname=f"training_set/{audio_filename}")
+                    
+                    # Add to the compiled metadata list
+                    compiled_metadata.append({
+                        "wav_filename": audio_filename,
+                        "label": row['label']
+                    })
+                
+                # 2. Test files (Even): Always included (Labels omitted from compiled CSV)
+                elif not is_training:
+                    audio_local = hf_hub_download(repo_id=DATASET_REPO_ID, filename=audio_repo_path, repo_type="dataset", token=HF_TOKEN)
+                    zipf.write(audio_local, arcname=f"test_set/{audio_filename}")
+                    
+                    # Add to compiled metadata but set label to HIDDEN or empty
+                    compiled_metadata.append({
+                        "wav_filename": audio_filename,
+                        "label": "HIDDEN"
+                    })
+
+            # --- CREATE THE SINGLE CONSOLIDATED CSV ---
+            if compiled_metadata:
+                master_df = pd.DataFrame(compiled_metadata)
+                master_csv_path = os.path.join(tmp_dir, "metadata_summary.csv")
+                # Save only the columns requested
+                master_df.to_csv(master_csv_path, index=False, columns=["wav_filename", "label"])
+                # Place it at the root of the ZIP for easy access
+                zipf.write(master_csv_path, arcname="metadata_summary.csv")
+
+        return zip_path, f"✅ ZIP created with {len(compiled_metadata)} total references."
+
+    except Exception as e:
+        return None, f"❌ Error: {str(e)}"
 
     # except Exception as e:
     #     return None, f"❌ Error creating ZIP: {str(e)}"
@@ -255,11 +319,11 @@ def access_dataset_zip(email, coupons_str):
 # --- UI ---
 
 with gr.Blocks() as demo:
-    gr.Markdown("# 🎙️ ENSIM Sound Data Platform")
+    gr.Markdown("# 🎙️ Sound Data Platform")
     
     with gr.Tabs():
         # STUDENT TAB
-        with gr.TabItem("Student Recording"):
+        with gr.TabItem("Dataset Collection"):
             with gr.Row():
                 email_input = gr.Textbox(label="Email", placeholder="test")
                 login_btn = gr.Button("Verify", variant="primary")
@@ -273,7 +337,7 @@ with gr.Blocks() as demo:
                 coupon_display = gr.Textbox(label="🎟️ YOUR COUPON (Save this!)", interactive=False)
 
         # 2. DATASET ACCESS TAB
-        with gr.TabItem("2. Dataset Access"):
+        with gr.TabItem("Dataset Access"):
             gr.Markdown("""
             ### 🔓 Unlock Your Data Partition
             - **Training Data:** You receive Training samples (Audio + Label) proportional to your coupons.
@@ -311,7 +375,12 @@ with gr.Blocks() as demo:
 
     # --- EVENT HANDLERS ---
     login_btn.click(verify_user, [email_input], [recording_zone, login_status])
-    submit_btn.click(upload_data, [email_input, label_input, audio_input], [res_msg, coupon_display])
+    
+    submit_btn.click(
+        fn=upload_data,
+        inputs=[email_input, label_input, audio_input],
+        outputs=[res_msg, audio_input, label_input, coupon_display]
+    )
     
     admin_login_btn.click(
         admin_login,