Support optional gzip compression for FASTQ output files to reduce size

pipeline.py

@@ -306,7 +306,8 @@ def generate_synthetic_bam(
     work_dir, snvs_bed, fasta_path, depth, vaf, rg_id, rg_sm,
     insert_size, insert_std, indel_interval, read_length=150,
     sequencing_mode="hybrid_capture",
-    log_func=None, progress_func=None
+    log_func=None, progress_func=None,
+    compress_fastq=False
 ):
     import pysam
     from collections import defaultdict
@@ -521,6 +522,19 @@ def generate_synthetic_bam(
     fastq_r2 = work_dir / "synthetic_R2.fastq"
     run_cmd(f"samtools fastq -1 {fastq_r1} -2 {fastq_r2} -0 {dev_null} -s {dev_null} {output_bam}", log_func)
 
+    if compress_fastq:
+        if progress_func:
+            progress_func(0.97, "Compressing FASTQ files (gzip)...")
+        import gzip
+        import shutil
+        log("Compressing FASTQ files to gzip format...", log_func)
+        for f_path in [fastq_r1, fastq_r2]:
+            gz_path = f_path.with_suffix(".fastq.gz")
+            with open(f_path, "rb") as f_in:
+                with gzip.open(gz_path, "wb", compresslevel=3) as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+            f_path.unlink()
+
     if progress_func:
         progress_func(1.0, "Sorting and indexing BAM...")
 

src/pipeline.py

@@ -306,7 +306,8 @@ def generate_synthetic_bam(
     work_dir, snvs_bed, fasta_path, depth, vaf, rg_id, rg_sm,
     insert_size, insert_std, indel_interval, read_length=150,
     sequencing_mode="hybrid_capture",
-    log_func=None, progress_func=None
+    log_func=None, progress_func=None,
+    compress_fastq=False
 ):
     import pysam
     from collections import defaultdict
@@ -521,6 +522,19 @@ def generate_synthetic_bam(
     fastq_r2 = work_dir / "synthetic_R2.fastq"
     run_cmd(f"samtools fastq -1 {fastq_r1} -2 {fastq_r2} -0 {dev_null} -s {dev_null} {output_bam}", log_func)
 
+    if compress_fastq:
+        if progress_func:
+            progress_func(0.97, "Compressing FASTQ files (gzip)...")
+        import gzip
+        import shutil
+        log("Compressing FASTQ files to gzip format...", log_func)
+        for f_path in [fastq_r1, fastq_r2]:
+            gz_path = f_path.with_suffix(".fastq.gz")
+            with open(f_path, "rb") as f_in:
+                with gzip.open(gz_path, "wb", compresslevel=3) as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+            f_path.unlink()
+
     if progress_func:
         progress_func(1.0, "Sorting and indexing BAM...")
 

src/streamlit_app.py

@@ -346,6 +346,13 @@ with st.sidebar:
     rg_id = st.text_input("Read Group ID", value="CPDV2510843-SEQ-251103")
     rg_sm = st.text_input("Sample Name", value="CPDV2510843-SEQ-251103")
     
+    st.subheader("FASTQ Output")
+    compress_fastq = st.checkbox(
+        "Gzip FASTQ output (.fastq.gz)",
+        value=True,
+        help="Compress the generated FASTQ files using gzip. This makes the files about 4-5x smaller, saving disk space and download time."
+    )
+    
     st.divider()
     st.subheader("🛠️ Debug Info")
     st.caption("Helpful diagnostics for troubleshooting deployment status.")
@@ -647,6 +654,7 @@ if run_btn:
             sequencing_mode="pcr_amplicon" if seq_mode.startswith("PCR Amplicon") else "hybrid_capture",
             log_func=append_log,
             progress_func=bam_progress,
+            compress_fastq=compress_fastq,
         )
 
         update_progress(1.0, "Done!")
@@ -662,8 +670,8 @@ if run_btn:
         static_dest_cwd = Path("static") / work_dir_name
         static_dest_script = Path(__file__).parent / "static" / work_dir_name
         
-        fastq_r1_path = work_dir / "synthetic_R1.fastq"
-        fastq_r2_path = work_dir / "synthetic_R2.fastq"
+        fastq_r1_path = work_dir / ("synthetic_R1.fastq.gz" if compress_fastq else "synthetic_R1.fastq")
+        fastq_r2_path = work_dir / ("synthetic_R2.fastq.gz" if compress_fastq else "synthetic_R2.fastq")
 
         for dest in [static_dest_cwd, static_dest_script]:
             dest.mkdir(parents=True, exist_ok=True)
@@ -677,9 +685,9 @@ if run_btn:
             if mane_transcripts_bed and mane_transcripts_bed.exists():
                 shutil.copy(mane_transcripts_bed, dest / "mane_transcripts.bed")
             if fastq_r1_path.exists():
-                shutil.copy(fastq_r1_path, dest / "synthetic_R1.fastq")
+                shutil.copy(fastq_r1_path, dest / fastq_r1_path.name)
             if fastq_r2_path.exists():
-                shutil.copy(fastq_r2_path, dest / "synthetic_R2.fastq")
+                shutil.copy(fastq_r2_path, dest / fastq_r2_path.name)
 
         # Store URLs and path strings — never load large files into session_state memory
         st.session_state["results"] = {
@@ -699,10 +707,11 @@ if run_btn:
             "igv_bed_url": f"/app/static/{work_dir_name}/igv_variant_navigator.bed",
             "fully_covered_bed_url": f"/app/static/{work_dir_name}/fully_covered_exons.bed" if fully_bed_path and fully_bed_path.exists() else None,
             "mane_transcripts_bed_url": f"/app/static/{work_dir_name}/mane_transcripts.bed" if mane_transcripts_bed and mane_transcripts_bed.exists() else None,
-            "fastq_r1_url": f"/app/static/{work_dir_name}/synthetic_R1.fastq" if fastq_r1_path.exists() else None,
-            "fastq_r2_url": f"/app/static/{work_dir_name}/synthetic_R2.fastq" if fastq_r2_path.exists() else None,
+            "fastq_r1_url": f"/app/static/{work_dir_name}/{fastq_r1_path.name}" if fastq_r1_path.exists() else None,
+            "fastq_r2_url": f"/app/static/{work_dir_name}/{fastq_r2_path.name}" if fastq_r2_path.exists() else None,
             "work_dir_name": work_dir_name,
             "genome_version": genome_version,
+            "compress_fastq": compress_fastq,
         }
         st.session_state["log_lines"] = log_lines[:]
 
@@ -861,8 +870,11 @@ if "results" in st.session_state:
 
     # Row 3 of download buttons (FASTQ R1/R2)
     dl7, dl8, dl9 = st.columns(3)
+    is_compressed = res.get("compress_fastq", False)
+    ext = ".fastq.gz" if is_compressed else ".fastq"
+
     with dl7:
-        st.markdown("**Synthetic FASTQ R1 (Forward)**")
+        st.markdown(f"**Synthetic FASTQ R1 (Forward){' (gzipped)' if is_compressed else ''}**")
         fastq_r1_path = Path(res["fastq_r1_path"]) if res.get("fastq_r1_path") else None
         if fastq_r1_path and fastq_r1_path.exists():
             if st.checkbox("Prepare FASTQ R1", key="prep_r1", help="Click to load R1 FASTQ file into memory to prepare it for download."):
@@ -870,12 +882,12 @@ if "results" in st.session_state:
                     st.download_button(
                         "⬇ Download FASTQ R1",
                         data=f,
-                        file_name="synthetic_R1.fastq",
+                        file_name=f"synthetic_R1{ext}",
                         mime="application/octet-stream",
                         use_container_width=True,
                     )
     with dl8:
-        st.markdown("**Synthetic FASTQ R2 (Reverse)**")
+        st.markdown(f"**Synthetic FASTQ R2 (Reverse){' (gzipped)' if is_compressed else ''}**")
         fastq_r2_path = Path(res["fastq_r2_path"]) if res.get("fastq_r2_path") else None
         if fastq_r2_path and fastq_r2_path.exists():
             if st.checkbox("Prepare FASTQ R2", key="prep_r2", help="Click to load R2 FASTQ file into memory to prepare it for download."):
@@ -883,7 +895,7 @@ if "results" in st.session_state:
                     st.download_button(
                         "⬇ Download FASTQ R2",
                         data=f,
-                        file_name="synthetic_R2.fastq",
+                        file_name=f"synthetic_R2{ext}",
                         mime="application/octet-stream",
                         use_container_width=True,
                     )

streamlit_app.py

@@ -346,6 +346,13 @@ with st.sidebar:
     rg_id = st.text_input("Read Group ID", value="CPDV2510843-SEQ-251103")
     rg_sm = st.text_input("Sample Name", value="CPDV2510843-SEQ-251103")
     
+    st.subheader("FASTQ Output")
+    compress_fastq = st.checkbox(
+        "Gzip FASTQ output (.fastq.gz)",
+        value=True,
+        help="Compress the generated FASTQ files using gzip. This makes the files about 4-5x smaller, saving disk space and download time."
+    )
+    
     st.divider()
     st.subheader("🛠️ Debug Info")
     st.caption("Helpful diagnostics for troubleshooting deployment status.")
@@ -647,6 +654,7 @@ if run_btn:
             sequencing_mode="pcr_amplicon" if seq_mode.startswith("PCR Amplicon") else "hybrid_capture",
             log_func=append_log,
             progress_func=bam_progress,
+            compress_fastq=compress_fastq,
         )
 
         update_progress(1.0, "Done!")
@@ -662,8 +670,8 @@ if run_btn:
         static_dest_cwd = Path("static") / work_dir_name
         static_dest_script = Path(__file__).parent / "static" / work_dir_name
         
-        fastq_r1_path = work_dir / "synthetic_R1.fastq"
-        fastq_r2_path = work_dir / "synthetic_R2.fastq"
+        fastq_r1_path = work_dir / ("synthetic_R1.fastq.gz" if compress_fastq else "synthetic_R1.fastq")
+        fastq_r2_path = work_dir / ("synthetic_R2.fastq.gz" if compress_fastq else "synthetic_R2.fastq")
 
         for dest in [static_dest_cwd, static_dest_script]:
             dest.mkdir(parents=True, exist_ok=True)
@@ -677,9 +685,9 @@ if run_btn:
             if mane_transcripts_bed and mane_transcripts_bed.exists():
                 shutil.copy(mane_transcripts_bed, dest / "mane_transcripts.bed")
             if fastq_r1_path.exists():
-                shutil.copy(fastq_r1_path, dest / "synthetic_R1.fastq")
+                shutil.copy(fastq_r1_path, dest / fastq_r1_path.name)
             if fastq_r2_path.exists():
-                shutil.copy(fastq_r2_path, dest / "synthetic_R2.fastq")
+                shutil.copy(fastq_r2_path, dest / fastq_r2_path.name)
 
         # Store URLs and path strings — never load large files into session_state memory
         st.session_state["results"] = {
@@ -699,10 +707,11 @@ if run_btn:
             "igv_bed_url": f"/app/static/{work_dir_name}/igv_variant_navigator.bed",
             "fully_covered_bed_url": f"/app/static/{work_dir_name}/fully_covered_exons.bed" if fully_bed_path and fully_bed_path.exists() else None,
             "mane_transcripts_bed_url": f"/app/static/{work_dir_name}/mane_transcripts.bed" if mane_transcripts_bed and mane_transcripts_bed.exists() else None,
-            "fastq_r1_url": f"/app/static/{work_dir_name}/synthetic_R1.fastq" if fastq_r1_path.exists() else None,
-            "fastq_r2_url": f"/app/static/{work_dir_name}/synthetic_R2.fastq" if fastq_r2_path.exists() else None,
+            "fastq_r1_url": f"/app/static/{work_dir_name}/{fastq_r1_path.name}" if fastq_r1_path.exists() else None,
+            "fastq_r2_url": f"/app/static/{work_dir_name}/{fastq_r2_path.name}" if fastq_r2_path.exists() else None,
             "work_dir_name": work_dir_name,
             "genome_version": genome_version,
+            "compress_fastq": compress_fastq,
         }
         st.session_state["log_lines"] = log_lines[:]
 
@@ -861,8 +870,11 @@ if "results" in st.session_state:
 
     # Row 3 of download buttons (FASTQ R1/R2)
     dl7, dl8, dl9 = st.columns(3)
+    is_compressed = res.get("compress_fastq", False)
+    ext = ".fastq.gz" if is_compressed else ".fastq"
+
     with dl7:
-        st.markdown("**Synthetic FASTQ R1 (Forward)**")
+        st.markdown(f"**Synthetic FASTQ R1 (Forward){' (gzipped)' if is_compressed else ''}**")
         fastq_r1_path = Path(res["fastq_r1_path"]) if res.get("fastq_r1_path") else None
         if fastq_r1_path and fastq_r1_path.exists():
             if st.checkbox("Prepare FASTQ R1", key="prep_r1", help="Click to load R1 FASTQ file into memory to prepare it for download."):
@@ -870,12 +882,12 @@ if "results" in st.session_state:
                     st.download_button(
                         "⬇ Download FASTQ R1",
                         data=f,
-                        file_name="synthetic_R1.fastq",
+                        file_name=f"synthetic_R1{ext}",
                         mime="application/octet-stream",
                         use_container_width=True,
                     )
     with dl8:
-        st.markdown("**Synthetic FASTQ R2 (Reverse)**")
+        st.markdown(f"**Synthetic FASTQ R2 (Reverse){' (gzipped)' if is_compressed else ''}**")
         fastq_r2_path = Path(res["fastq_r2_path"]) if res.get("fastq_r2_path") else None
         if fastq_r2_path and fastq_r2_path.exists():
             if st.checkbox("Prepare FASTQ R2", key="prep_r2", help="Click to load R2 FASTQ file into memory to prepare it for download."):
@@ -883,7 +895,7 @@ if "results" in st.session_state:
                     st.download_button(
                         "⬇ Download FASTQ R2",
                         data=f,
-                        file_name="synthetic_R2.fastq",
+                        file_name=f"synthetic_R2{ext}",
                         mime="application/octet-stream",
                         use_container_width=True,
                     )