Spaces:
Sleeping
Sleeping
Halper-Stromberg commited on
Commit Β·
6a503f7
1
Parent(s): f72768b
Add Batch Controls Mode supporting fixed variant coordinates and customizable variant types, VAF, and indel sizes
Browse files- pipeline.py +22 -12
- src/pipeline.py +22 -12
- src/streamlit_app.py +220 -75
- streamlit_app.py +220 -75
pipeline.py
CHANGED
|
@@ -307,7 +307,9 @@ def generate_synthetic_bam(
|
|
| 307 |
insert_size, insert_std, indel_interval, read_length=150,
|
| 308 |
sequencing_mode="hybrid_capture",
|
| 309 |
log_func=None, progress_func=None,
|
| 310 |
-
compress_fastq=False
|
|
|
|
|
|
|
| 311 |
):
|
| 312 |
import pysam
|
| 313 |
from collections import defaultdict
|
|
@@ -344,18 +346,26 @@ def generate_synthetic_bam(
|
|
| 344 |
ref_base = fasta.fetch(chrom, pos, pos + 1).upper()
|
| 345 |
except Exception:
|
| 346 |
continue
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
else:
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 359 |
alt_bases = [b for b in ["A", "C", "G", "T"] if b != ref_base]
|
| 360 |
alt_seq = random.choice(alt_bases) if alt_bases else "A"
|
| 361 |
ref_seq = ref_base
|
|
|
|
| 307 |
insert_size, insert_std, indel_interval, read_length=150,
|
| 308 |
sequencing_mode="hybrid_capture",
|
| 309 |
log_func=None, progress_func=None,
|
| 310 |
+
compress_fastq=False,
|
| 311 |
+
variant_type="Mixed",
|
| 312 |
+
indel_length=0
|
| 313 |
):
|
| 314 |
import pysam
|
| 315 |
from collections import defaultdict
|
|
|
|
| 346 |
ref_base = fasta.fetch(chrom, pos, pos + 1).upper()
|
| 347 |
except Exception:
|
| 348 |
continue
|
| 349 |
+
# Determine variant type for this locus
|
| 350 |
+
current_type = variant_type
|
| 351 |
+
if variant_type == "Mixed":
|
| 352 |
+
if indel_interval > 0 and (variant_count + 1) % indel_interval == 0:
|
| 353 |
+
current_type = random.choice(["Insertion", "Deletion"])
|
| 354 |
else:
|
| 355 |
+
current_type = "SNV"
|
| 356 |
+
|
| 357 |
+
if current_type == "Insertion":
|
| 358 |
+
curr_len = indel_length if indel_length > 0 else random.randint(1, 4)
|
| 359 |
+
ins_bases = "".join(random.choices(["A", "C", "G", "T"], k=curr_len))
|
| 360 |
+
ref_seq, alt_seq = ref_base, ref_base + ins_bases
|
| 361 |
+
elif current_type == "Deletion":
|
| 362 |
+
curr_len = indel_length if indel_length > 0 else random.randint(1, 4)
|
| 363 |
+
try:
|
| 364 |
+
ref_seq = fasta.fetch(chrom, pos, pos + curr_len + 1).upper()
|
| 365 |
+
except Exception:
|
| 366 |
+
ref_seq = ref_base
|
| 367 |
+
alt_seq = ref_base
|
| 368 |
+
else: # SNV
|
| 369 |
alt_bases = [b for b in ["A", "C", "G", "T"] if b != ref_base]
|
| 370 |
alt_seq = random.choice(alt_bases) if alt_bases else "A"
|
| 371 |
ref_seq = ref_base
|
src/pipeline.py
CHANGED
|
@@ -307,7 +307,9 @@ def generate_synthetic_bam(
|
|
| 307 |
insert_size, insert_std, indel_interval, read_length=150,
|
| 308 |
sequencing_mode="hybrid_capture",
|
| 309 |
log_func=None, progress_func=None,
|
| 310 |
-
compress_fastq=False
|
|
|
|
|
|
|
| 311 |
):
|
| 312 |
import pysam
|
| 313 |
from collections import defaultdict
|
|
@@ -344,18 +346,26 @@ def generate_synthetic_bam(
|
|
| 344 |
ref_base = fasta.fetch(chrom, pos, pos + 1).upper()
|
| 345 |
except Exception:
|
| 346 |
continue
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
else:
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 359 |
alt_bases = [b for b in ["A", "C", "G", "T"] if b != ref_base]
|
| 360 |
alt_seq = random.choice(alt_bases) if alt_bases else "A"
|
| 361 |
ref_seq = ref_base
|
|
|
|
| 307 |
insert_size, insert_std, indel_interval, read_length=150,
|
| 308 |
sequencing_mode="hybrid_capture",
|
| 309 |
log_func=None, progress_func=None,
|
| 310 |
+
compress_fastq=False,
|
| 311 |
+
variant_type="Mixed",
|
| 312 |
+
indel_length=0
|
| 313 |
):
|
| 314 |
import pysam
|
| 315 |
from collections import defaultdict
|
|
|
|
| 346 |
ref_base = fasta.fetch(chrom, pos, pos + 1).upper()
|
| 347 |
except Exception:
|
| 348 |
continue
|
| 349 |
+
# Determine variant type for this locus
|
| 350 |
+
current_type = variant_type
|
| 351 |
+
if variant_type == "Mixed":
|
| 352 |
+
if indel_interval > 0 and (variant_count + 1) % indel_interval == 0:
|
| 353 |
+
current_type = random.choice(["Insertion", "Deletion"])
|
| 354 |
else:
|
| 355 |
+
current_type = "SNV"
|
| 356 |
+
|
| 357 |
+
if current_type == "Insertion":
|
| 358 |
+
curr_len = indel_length if indel_length > 0 else random.randint(1, 4)
|
| 359 |
+
ins_bases = "".join(random.choices(["A", "C", "G", "T"], k=curr_len))
|
| 360 |
+
ref_seq, alt_seq = ref_base, ref_base + ins_bases
|
| 361 |
+
elif current_type == "Deletion":
|
| 362 |
+
curr_len = indel_length if indel_length > 0 else random.randint(1, 4)
|
| 363 |
+
try:
|
| 364 |
+
ref_seq = fasta.fetch(chrom, pos, pos + curr_len + 1).upper()
|
| 365 |
+
except Exception:
|
| 366 |
+
ref_seq = ref_base
|
| 367 |
+
alt_seq = ref_base
|
| 368 |
+
else: # SNV
|
| 369 |
alt_bases = [b for b in ["A", "C", "G", "T"] if b != ref_base]
|
| 370 |
alt_seq = random.choice(alt_bases) if alt_bases else "A"
|
| 371 |
ref_seq = ref_base
|
src/streamlit_app.py
CHANGED
|
@@ -254,9 +254,18 @@ uploaded_bed = st.session_state.get("uploaded_bed_file")
|
|
| 254 |
with st.sidebar:
|
| 255 |
st.header("Pipeline Parameters")
|
| 256 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 257 |
st.subheader("Sequencing Parameters")
|
| 258 |
depth = st.number_input("Target read depth per variant", min_value=1, max_value=10000, value=100, step=10)
|
| 259 |
-
|
|
|
|
|
|
|
|
|
|
| 260 |
read_length = st.number_input("Read length (bp)", min_value=50, max_value=300, value=150, step=10)
|
| 261 |
|
| 262 |
st.subheader("Sequencing Technology")
|
|
@@ -279,9 +288,28 @@ with st.sidebar:
|
|
| 279 |
indel_interval = st.number_input(
|
| 280 |
"Indel interval (0 = SNVs only)",
|
| 281 |
min_value=0, max_value=100, value=10, step=1,
|
| 282 |
-
help="Make every Nth variant an indel. Set to 0 to generate only SNVs.",
|
| 283 |
)
|
| 284 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
st.divider()
|
| 286 |
st.subheader("Reference Genome")
|
| 287 |
genome_version = st.selectbox(
|
|
@@ -479,6 +507,32 @@ if uploaded_bed:
|
|
| 479 |
|
| 480 |
total_selected = len(edited_df[edited_df["Select"] == True])
|
| 481 |
st.info(f"Selected {total_selected:,} of {len(df):,} probes ({total_selected/len(df)*100:.1f}%) for variant generation.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 482 |
|
| 483 |
st.divider()
|
| 484 |
|
|
@@ -627,91 +681,165 @@ if run_btn:
|
|
| 627 |
log_func=append_log
|
| 628 |
)
|
| 629 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 630 |
append_log("\n============================================")
|
| 631 |
append_log(" VARIANT SUMMARY ")
|
| 632 |
append_log("============================================")
|
| 633 |
-
append_log(f"Total
|
|
|
|
| 634 |
append_log("============================================")
|
| 635 |
|
| 636 |
-
|
| 637 |
-
|
| 638 |
-
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
|
| 642 |
-
sorted_bam, output_vcf = pl.generate_synthetic_bam(
|
| 643 |
-
work_dir=work_dir,
|
| 644 |
-
snvs_bed=snvs_bed,
|
| 645 |
-
fasta_path=fasta_path,
|
| 646 |
-
depth=depth,
|
| 647 |
-
vaf=vaf,
|
| 648 |
-
rg_id=rg_id,
|
| 649 |
-
rg_sm=rg_sm,
|
| 650 |
-
insert_size=insert_size,
|
| 651 |
-
insert_std=insert_std,
|
| 652 |
-
indel_interval=indel_interval,
|
| 653 |
-
read_length=read_length,
|
| 654 |
-
sequencing_mode="pcr_amplicon" if seq_mode.startswith("PCR Amplicon") else "hybrid_capture",
|
| 655 |
-
log_func=append_log,
|
| 656 |
-
progress_func=bam_progress,
|
| 657 |
-
compress_fastq=compress_fastq,
|
| 658 |
-
)
|
| 659 |
-
|
| 660 |
-
update_progress(1.0, "Done!")
|
| 661 |
-
append_log("\nβ
Pipeline complete.")
|
| 662 |
|
| 663 |
-
bai_path = Path(str(sorted_bam) + ".bai")
|
| 664 |
-
vcf_path = Path(output_vcf) if not isinstance(output_vcf, Path) else output_vcf
|
| 665 |
igv_bed_path = Path(snvs_bed) if not isinstance(snvs_bed, Path) else snvs_bed
|
| 666 |
fully_bed_path = Path(fully_bed) if fully_bed and not isinstance(fully_bed, Path) else fully_bed
|
|
|
|
| 667 |
|
| 668 |
-
|
| 669 |
-
|
| 670 |
-
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
|
| 674 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 675 |
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 679 |
if bai_path.exists():
|
| 680 |
-
shutil.
|
| 681 |
-
shutil.
|
| 682 |
-
shutil.copy(igv_bed_path, dest / "igv_variant_navigator.bed")
|
| 683 |
-
if fully_bed_path and fully_bed_path.exists():
|
| 684 |
-
shutil.copy(fully_bed_path, dest / "fully_covered_exons.bed")
|
| 685 |
-
if mane_transcripts_bed and mane_transcripts_bed.exists():
|
| 686 |
-
shutil.copy(mane_transcripts_bed, dest / "mane_transcripts.bed")
|
| 687 |
if fastq_r1_path.exists():
|
| 688 |
-
shutil.
|
| 689 |
if fastq_r2_path.exists():
|
| 690 |
-
shutil.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 691 |
|
| 692 |
# Store URLs and path strings β never load large files into session_state memory
|
| 693 |
st.session_state["results"] = {
|
| 694 |
-
"
|
| 695 |
-
"
|
| 696 |
-
"bam_path": str(sorted_bam),
|
| 697 |
-
"bai_path": str(bai_path) if bai_path.exists() else None,
|
| 698 |
-
"vcf_path": str(vcf_path),
|
| 699 |
-
"igv_bed_path": str(igv_bed_path),
|
| 700 |
-
"fully_covered_bed_path": str(fully_bed_path) if fully_bed_path else None,
|
| 701 |
-
"mane_transcripts_bed_path": str(mane_transcripts_bed) if mane_transcripts_bed else None,
|
| 702 |
-
"fastq_r1_path": str(fastq_r1_path) if fastq_r1_path.exists() else None,
|
| 703 |
-
"fastq_r2_path": str(fastq_r2_path) if fastq_r2_path.exists() else None,
|
| 704 |
-
"bam_url": f"/app/static/{work_dir_name}/synthetic.sorted.bam",
|
| 705 |
-
"bai_url": f"/app/static/{work_dir_name}/synthetic.sorted.bam.bai" if bai_path.exists() else None,
|
| 706 |
-
"vcf_url": f"/app/static/{work_dir_name}/synthetic.vcf",
|
| 707 |
-
"igv_bed_url": f"/app/static/{work_dir_name}/igv_variant_navigator.bed",
|
| 708 |
-
"fully_covered_bed_url": f"/app/static/{work_dir_name}/fully_covered_exons.bed" if fully_bed_path and fully_bed_path.exists() else None,
|
| 709 |
-
"mane_transcripts_bed_url": f"/app/static/{work_dir_name}/mane_transcripts.bed" if mane_transcripts_bed and mane_transcripts_bed.exists() else None,
|
| 710 |
-
"fastq_r1_url": f"/app/static/{work_dir_name}/{fastq_r1_path.name}" if fastq_r1_path.exists() else None,
|
| 711 |
-
"fastq_r2_url": f"/app/static/{work_dir_name}/{fastq_r2_path.name}" if fastq_r2_path.exists() else None,
|
| 712 |
-
"work_dir_name": work_dir_name,
|
| 713 |
-
"genome_version": genome_version,
|
| 714 |
-
"compress_fastq": compress_fastq,
|
| 715 |
}
|
| 716 |
st.session_state["log_lines"] = log_lines[:]
|
| 717 |
|
|
@@ -723,10 +851,8 @@ if run_btn:
|
|
| 723 |
# ββ Results section (persists across reruns via session_state) ββββββββββββββββ
|
| 724 |
|
| 725 |
if "results" in st.session_state:
|
| 726 |
-
|
| 727 |
-
|
| 728 |
-
total_snvs = res["total_snvs"]
|
| 729 |
-
|
| 730 |
st.success("Pipeline completed successfully!")
|
| 731 |
|
| 732 |
# Show log if available and pipeline didn't just run
|
|
@@ -735,6 +861,25 @@ if "results" in st.session_state:
|
|
| 735 |
st.code("\n".join(st.session_state["log_lines"][-80:]), language=None)
|
| 736 |
|
| 737 |
st.header("3 Β· Results")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 738 |
m1, m2, m3, m4 = st.columns(4)
|
| 739 |
m1.metric("Fully Covered Exons", f"{stats['fully_covered']:,}")
|
| 740 |
m2.metric("Partially Covered Exons", f"{stats['partially_covered']:,}")
|
|
|
|
| 254 |
with st.sidebar:
|
| 255 |
st.header("Pipeline Parameters")
|
| 256 |
|
| 257 |
+
enable_batch = st.checkbox(
|
| 258 |
+
"Enable Batch Controls Mode",
|
| 259 |
+
value=False,
|
| 260 |
+
help="Enable this to generate multiple in-silico controls (differing by variant type/VAF) at the exact same coordinates in a single run."
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
st.subheader("Sequencing Parameters")
|
| 264 |
depth = st.number_input("Target read depth per variant", min_value=1, max_value=10000, value=100, step=10)
|
| 265 |
+
if not enable_batch:
|
| 266 |
+
vaf = st.slider("Variant allele frequency (VAF)", min_value=0.01, max_value=1.0, value=0.20, step=0.01, format="%.2f")
|
| 267 |
+
else:
|
| 268 |
+
vaf = 0.20
|
| 269 |
read_length = st.number_input("Read length (bp)", min_value=50, max_value=300, value=150, step=10)
|
| 270 |
|
| 271 |
st.subheader("Sequencing Technology")
|
|
|
|
| 288 |
indel_interval = st.number_input(
|
| 289 |
"Indel interval (0 = SNVs only)",
|
| 290 |
min_value=0, max_value=100, value=10, step=1,
|
| 291 |
+
help="Make every Nth variant an indel. Set to 0 to generate only SNVs. Only active when using Mixed (Random) variants.",
|
| 292 |
)
|
| 293 |
|
| 294 |
+
st.subheader("Variant Specification")
|
| 295 |
+
if not enable_batch:
|
| 296 |
+
var_type = st.selectbox(
|
| 297 |
+
"Variant type",
|
| 298 |
+
options=["SNV", "Insertion", "Deletion", "Mixed (Random)"],
|
| 299 |
+
index=3,
|
| 300 |
+
help="Choose the type of variants to generate. Mixed (Random) alternates between SNVs and Indels."
|
| 301 |
+
)
|
| 302 |
+
if var_type in ["Insertion", "Deletion"]:
|
| 303 |
+
var_length = st.number_input(
|
| 304 |
+
"Indel length (bp)",
|
| 305 |
+
min_value=1, max_value=100, value=3, step=1,
|
| 306 |
+
help="The exact size of the insertion or deletion in base pairs."
|
| 307 |
+
)
|
| 308 |
+
else:
|
| 309 |
+
var_length = 0
|
| 310 |
+
else:
|
| 311 |
+
st.info("βΉοΈ Batch mode enabled. Variant specifications are configured in the batch table in the main panel.")
|
| 312 |
+
|
| 313 |
st.divider()
|
| 314 |
st.subheader("Reference Genome")
|
| 315 |
genome_version = st.selectbox(
|
|
|
|
| 507 |
|
| 508 |
total_selected = len(edited_df[edited_df["Select"] == True])
|
| 509 |
st.info(f"Selected {total_selected:,} of {len(df):,} probes ({total_selected/len(df)*100:.1f}%) for variant generation.")
|
| 510 |
+
if uploaded_bed and enable_batch:
|
| 511 |
+
st.header("1.8 Β· Configure Batch Runs")
|
| 512 |
+
st.caption("Define the list of controls to generate. All runs will share the same variant coordinates.")
|
| 513 |
+
|
| 514 |
+
import pandas as pd
|
| 515 |
+
default_batch = pd.DataFrame([
|
| 516 |
+
{"Control Name": "control_snv", "VAF": 0.20, "Variant Type": "SNV", "Indel Length": 0},
|
| 517 |
+
{"Control Name": "control_ins_3bp", "VAF": 0.20, "Variant Type": "Insertion", "Indel Length": 3},
|
| 518 |
+
{"Control Name": "control_del_5bp", "VAF": 0.20, "Variant Type": "Deletion", "Indel Length": 5},
|
| 519 |
+
])
|
| 520 |
+
|
| 521 |
+
if "batch_df" not in st.session_state:
|
| 522 |
+
st.session_state["batch_df"] = default_batch
|
| 523 |
+
|
| 524 |
+
batch_df = st.data_editor(
|
| 525 |
+
st.session_state["batch_df"],
|
| 526 |
+
num_rows="dynamic",
|
| 527 |
+
use_container_width=True,
|
| 528 |
+
column_config={
|
| 529 |
+
"Control Name": st.column_config.TextColumn("Control Name", required=True),
|
| 530 |
+
"VAF": st.column_config.NumberColumn("VAF", min_value=0.01, max_value=1.0, step=0.01, format="%.2f", required=True),
|
| 531 |
+
"Variant Type": st.column_config.SelectboxColumn("Variant Type", options=["SNV", "Insertion", "Deletion", "Mixed (Random)"], required=True),
|
| 532 |
+
"Indel Length": st.column_config.NumberColumn("Indel Length (bp)", min_value=0, max_value=100, step=1, default=0),
|
| 533 |
+
}
|
| 534 |
+
)
|
| 535 |
+
st.session_state["batch_df"] = batch_df
|
| 536 |
|
| 537 |
st.divider()
|
| 538 |
|
|
|
|
| 681 |
log_func=append_log
|
| 682 |
)
|
| 683 |
|
| 684 |
+
# Resolve runs list
|
| 685 |
+
if enable_batch:
|
| 686 |
+
runs_to_process = []
|
| 687 |
+
for _, row in batch_df.iterrows():
|
| 688 |
+
runs_to_process.append({
|
| 689 |
+
"name": row["Control Name"].strip(),
|
| 690 |
+
"vaf": float(row["VAF"]),
|
| 691 |
+
"type": "Mixed" if row["Variant Type"].startswith("Mixed") else row["Variant Type"],
|
| 692 |
+
"length": int(row["Indel Length"]),
|
| 693 |
+
})
|
| 694 |
+
else:
|
| 695 |
+
runs_to_process = [{
|
| 696 |
+
"name": "Single Control",
|
| 697 |
+
"vaf": vaf,
|
| 698 |
+
"type": "Mixed" if var_type.startswith("Mixed") else var_type,
|
| 699 |
+
"length": var_length,
|
| 700 |
+
}]
|
| 701 |
+
|
| 702 |
+
# Sanitize and deduplicate names
|
| 703 |
+
import re
|
| 704 |
+
seen_names = set()
|
| 705 |
+
sanitized_runs = []
|
| 706 |
+
for r in runs_to_process:
|
| 707 |
+
clean_name = re.sub(r'[^a-zA-Z0-9_]', '_', r["name"])
|
| 708 |
+
if not clean_name:
|
| 709 |
+
clean_name = "control"
|
| 710 |
+
original_clean = clean_name
|
| 711 |
+
counter = 1
|
| 712 |
+
while clean_name in seen_names:
|
| 713 |
+
clean_name = f"{original_clean}_{counter}"
|
| 714 |
+
counter += 1
|
| 715 |
+
seen_names.add(clean_name)
|
| 716 |
+
r["name"] = clean_name
|
| 717 |
+
sanitized_runs.append(r)
|
| 718 |
+
runs_to_process = sanitized_runs
|
| 719 |
+
|
| 720 |
append_log("\n============================================")
|
| 721 |
append_log(" VARIANT SUMMARY ")
|
| 722 |
append_log("============================================")
|
| 723 |
+
append_log(f"Total SNV loci generated: {total_snvs}")
|
| 724 |
+
append_log(f"Total runs to execute: {len(runs_to_process)}")
|
| 725 |
append_log("============================================")
|
| 726 |
|
| 727 |
+
runs_results = {}
|
| 728 |
+
total_runs = len(runs_to_process)
|
| 729 |
+
work_dir_name = work_dir.name
|
| 730 |
+
static_dest_cwd = Path("static") / work_dir_name
|
| 731 |
+
static_dest_script = Path(__file__).parent / "static" / work_dir_name
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 732 |
|
|
|
|
|
|
|
| 733 |
igv_bed_path = Path(snvs_bed) if not isinstance(snvs_bed, Path) else snvs_bed
|
| 734 |
fully_bed_path = Path(fully_bed) if fully_bed and not isinstance(fully_bed, Path) else fully_bed
|
| 735 |
+
mane_transcripts_bed_path = Path(mane_transcripts_bed) if mane_transcripts_bed and not isinstance(mane_transcripts_bed, Path) else mane_transcripts_bed
|
| 736 |
|
| 737 |
+
for run_idx, run in enumerate(runs_to_process):
|
| 738 |
+
run_name = run["name"]
|
| 739 |
+
run_vaf = run["vaf"]
|
| 740 |
+
run_type = run["type"]
|
| 741 |
+
run_length = run["length"]
|
| 742 |
+
|
| 743 |
+
append_log(f"\n=== Generating Synthetic BAM for {run_name} (VAF={run_vaf}, {run_type}) ===")
|
| 744 |
+
|
| 745 |
+
def bam_progress(fraction, label):
|
| 746 |
+
update_progress(0.40 + (run_idx + fraction) / total_runs * 0.55, f"[{run_name}] {label}")
|
| 747 |
+
|
| 748 |
+
sorted_bam, output_vcf = pl.generate_synthetic_bam(
|
| 749 |
+
work_dir=work_dir,
|
| 750 |
+
snvs_bed=snvs_bed,
|
| 751 |
+
fasta_path=fasta_path,
|
| 752 |
+
depth=depth,
|
| 753 |
+
vaf=run_vaf,
|
| 754 |
+
rg_id=rg_id,
|
| 755 |
+
rg_sm=rg_sm,
|
| 756 |
+
insert_size=insert_size,
|
| 757 |
+
insert_std=insert_std,
|
| 758 |
+
indel_interval=indel_interval,
|
| 759 |
+
read_length=read_length,
|
| 760 |
+
sequencing_mode="pcr_amplicon" if seq_mode.startswith("PCR Amplicon") else "hybrid_capture",
|
| 761 |
+
log_func=append_log,
|
| 762 |
+
progress_func=bam_progress,
|
| 763 |
+
compress_fastq=compress_fastq,
|
| 764 |
+
variant_type=run_type,
|
| 765 |
+
indel_length=run_length
|
| 766 |
+
)
|
| 767 |
|
| 768 |
+
# Resolve output paths in work_dir
|
| 769 |
+
bai_path = Path(str(sorted_bam) + ".bai")
|
| 770 |
+
vcf_path = Path(output_vcf) if not isinstance(output_vcf, Path) else output_vcf
|
| 771 |
+
fastq_r1_path = work_dir / ("synthetic_R1.fastq.gz" if compress_fastq else "synthetic_R1.fastq")
|
| 772 |
+
fastq_r2_path = work_dir / ("synthetic_R2.fastq.gz" if compress_fastq else "synthetic_R2.fastq")
|
| 773 |
+
|
| 774 |
+
# Relocate to run subfolder in work_dir
|
| 775 |
+
run_work_dir = work_dir / run_name
|
| 776 |
+
run_work_dir.mkdir(parents=True, exist_ok=True)
|
| 777 |
+
|
| 778 |
+
bam_reloc = run_work_dir / "synthetic.sorted.bam"
|
| 779 |
+
bai_reloc = run_work_dir / "synthetic.sorted.bam.bai"
|
| 780 |
+
vcf_reloc = run_work_dir / "synthetic.vcf"
|
| 781 |
+
fastq_r1_reloc = run_work_dir / fastq_r1_path.name
|
| 782 |
+
fastq_r2_reloc = run_work_dir / fastq_r2_path.name
|
| 783 |
+
|
| 784 |
+
shutil.move(str(sorted_bam), str(bam_reloc))
|
| 785 |
if bai_path.exists():
|
| 786 |
+
shutil.move(str(bai_path), str(bai_reloc))
|
| 787 |
+
shutil.move(str(vcf_path), str(vcf_reloc))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 788 |
if fastq_r1_path.exists():
|
| 789 |
+
shutil.move(str(fastq_r1_path), str(fastq_r1_reloc))
|
| 790 |
if fastq_r2_path.exists():
|
| 791 |
+
shutil.move(str(fastq_r2_path), str(fastq_r2_reloc))
|
| 792 |
+
|
| 793 |
+
# Copy to static subfolder dest / run_name
|
| 794 |
+
for dest_root in [static_dest_cwd, static_dest_script]:
|
| 795 |
+
dest = dest_root / run_name
|
| 796 |
+
dest.mkdir(parents=True, exist_ok=True)
|
| 797 |
+
|
| 798 |
+
shutil.copy(bam_reloc, dest / "synthetic.sorted.bam")
|
| 799 |
+
if bai_reloc.exists():
|
| 800 |
+
shutil.copy(bai_reloc, dest / "synthetic.sorted.bam.bai")
|
| 801 |
+
shutil.copy(vcf_reloc, dest / "synthetic.vcf")
|
| 802 |
+
shutil.copy(igv_bed_path, dest / "igv_variant_navigator.bed")
|
| 803 |
+
if fully_bed_path and fully_bed_path.exists():
|
| 804 |
+
shutil.copy(fully_bed_path, dest / "fully_covered_exons.bed")
|
| 805 |
+
if mane_transcripts_bed_path and mane_transcripts_bed_path.exists():
|
| 806 |
+
shutil.copy(mane_transcripts_bed_path, dest / "mane_transcripts.bed")
|
| 807 |
+
if fastq_r1_reloc.exists():
|
| 808 |
+
shutil.copy(fastq_r1_reloc, dest / fastq_r1_reloc.name)
|
| 809 |
+
if fastq_r2_reloc.exists():
|
| 810 |
+
shutil.copy(fastq_r2_reloc, dest / fastq_r2_reloc.name)
|
| 811 |
+
|
| 812 |
+
runs_results[run_name] = {
|
| 813 |
+
"stats": stats,
|
| 814 |
+
"total_snvs": total_snvs,
|
| 815 |
+
"bam_path": str(bam_reloc),
|
| 816 |
+
"bai_path": str(bai_reloc) if bai_reloc.exists() else None,
|
| 817 |
+
"vcf_path": str(vcf_reloc),
|
| 818 |
+
"igv_bed_path": str(igv_bed_path),
|
| 819 |
+
"fully_covered_bed_path": str(fully_bed_path) if fully_bed_path else None,
|
| 820 |
+
"mane_transcripts_bed_path": str(mane_transcripts_bed_path) if mane_transcripts_bed_path else None,
|
| 821 |
+
"fastq_r1_path": str(fastq_r1_reloc) if fastq_r1_reloc.exists() else None,
|
| 822 |
+
"fastq_r2_path": str(fastq_r2_reloc) if fastq_r2_reloc.exists() else None,
|
| 823 |
+
"bam_url": f"/app/static/{work_dir_name}/{run_name}/synthetic.sorted.bam",
|
| 824 |
+
"bai_url": f"/app/static/{work_dir_name}/{run_name}/synthetic.sorted.bam.bai" if bai_reloc.exists() else None,
|
| 825 |
+
"vcf_url": f"/app/static/{work_dir_name}/{run_name}/synthetic.vcf",
|
| 826 |
+
"igv_bed_url": f"/app/static/{work_dir_name}/{run_name}/igv_variant_navigator.bed",
|
| 827 |
+
"fully_covered_bed_url": f"/app/static/{work_dir_name}/{run_name}/fully_covered_exons.bed" if fully_bed_path and fully_bed_path.exists() else None,
|
| 828 |
+
"mane_transcripts_bed_url": f"/app/static/{work_dir_name}/{run_name}/mane_transcripts.bed" if mane_transcripts_bed_path and mane_transcripts_bed_path.exists() else None,
|
| 829 |
+
"fastq_r1_url": f"/app/static/{work_dir_name}/{run_name}/{fastq_r1_reloc.name}" if fastq_r1_reloc.exists() else None,
|
| 830 |
+
"fastq_r2_url": f"/app/static/{work_dir_name}/{run_name}/{fastq_r2_reloc.name}" if fastq_r2_reloc.exists() else None,
|
| 831 |
+
"work_dir_name": f"{work_dir_name}/{run_name}",
|
| 832 |
+
"genome_version": genome_version,
|
| 833 |
+
"compress_fastq": compress_fastq,
|
| 834 |
+
}
|
| 835 |
+
|
| 836 |
+
update_progress(1.0, "Done!")
|
| 837 |
+
append_log("\nβ
Pipeline complete.")
|
| 838 |
|
| 839 |
# Store URLs and path strings β never load large files into session_state memory
|
| 840 |
st.session_state["results"] = {
|
| 841 |
+
"is_batch": enable_batch,
|
| 842 |
+
"runs": runs_results
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 843 |
}
|
| 844 |
st.session_state["log_lines"] = log_lines[:]
|
| 845 |
|
|
|
|
| 851 |
# ββ Results section (persists across reruns via session_state) ββββββββββββββββ
|
| 852 |
|
| 853 |
if "results" in st.session_state:
|
| 854 |
+
raw_results = st.session_state["results"]
|
| 855 |
+
|
|
|
|
|
|
|
| 856 |
st.success("Pipeline completed successfully!")
|
| 857 |
|
| 858 |
# Show log if available and pipeline didn't just run
|
|
|
|
| 861 |
st.code("\n".join(st.session_state["log_lines"][-80:]), language=None)
|
| 862 |
|
| 863 |
st.header("3 Β· Results")
|
| 864 |
+
|
| 865 |
+
is_batch = raw_results.get("is_batch", False)
|
| 866 |
+
if is_batch:
|
| 867 |
+
run_names = list(raw_results["runs"].keys())
|
| 868 |
+
selected_run = st.selectbox(
|
| 869 |
+
"Select control run to view/download",
|
| 870 |
+
options=run_names,
|
| 871 |
+
help="Choose which batch control configuration to display and download outputs for."
|
| 872 |
+
)
|
| 873 |
+
res = raw_results["runs"][selected_run]
|
| 874 |
+
|
| 875 |
+
# Display run details
|
| 876 |
+
st.markdown(f"**Selected Run Parameters:** Variant Type: `{res.get('variant_type')}`, VAF: `{res.get('vaf')}`" +
|
| 877 |
+
(f", Indel Length: `{res.get('indel_length')} bp`" if res.get("variant_type") in ["Insertion", "Deletion"] else ""))
|
| 878 |
+
else:
|
| 879 |
+
res = list(raw_results["runs"].values())[0]
|
| 880 |
+
|
| 881 |
+
stats = res["stats"]
|
| 882 |
+
total_snvs = res["total_snvs"]
|
| 883 |
m1, m2, m3, m4 = st.columns(4)
|
| 884 |
m1.metric("Fully Covered Exons", f"{stats['fully_covered']:,}")
|
| 885 |
m2.metric("Partially Covered Exons", f"{stats['partially_covered']:,}")
|
streamlit_app.py
CHANGED
|
@@ -254,9 +254,18 @@ uploaded_bed = st.session_state.get("uploaded_bed_file")
|
|
| 254 |
with st.sidebar:
|
| 255 |
st.header("Pipeline Parameters")
|
| 256 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 257 |
st.subheader("Sequencing Parameters")
|
| 258 |
depth = st.number_input("Target read depth per variant", min_value=1, max_value=10000, value=100, step=10)
|
| 259 |
-
|
|
|
|
|
|
|
|
|
|
| 260 |
read_length = st.number_input("Read length (bp)", min_value=50, max_value=300, value=150, step=10)
|
| 261 |
|
| 262 |
st.subheader("Sequencing Technology")
|
|
@@ -279,9 +288,28 @@ with st.sidebar:
|
|
| 279 |
indel_interval = st.number_input(
|
| 280 |
"Indel interval (0 = SNVs only)",
|
| 281 |
min_value=0, max_value=100, value=10, step=1,
|
| 282 |
-
help="Make every Nth variant an indel. Set to 0 to generate only SNVs.",
|
| 283 |
)
|
| 284 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
st.divider()
|
| 286 |
st.subheader("Reference Genome")
|
| 287 |
genome_version = st.selectbox(
|
|
@@ -479,6 +507,32 @@ if uploaded_bed:
|
|
| 479 |
|
| 480 |
total_selected = len(edited_df[edited_df["Select"] == True])
|
| 481 |
st.info(f"Selected {total_selected:,} of {len(df):,} probes ({total_selected/len(df)*100:.1f}%) for variant generation.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 482 |
|
| 483 |
st.divider()
|
| 484 |
|
|
@@ -627,91 +681,165 @@ if run_btn:
|
|
| 627 |
log_func=append_log
|
| 628 |
)
|
| 629 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 630 |
append_log("\n============================================")
|
| 631 |
append_log(" VARIANT SUMMARY ")
|
| 632 |
append_log("============================================")
|
| 633 |
-
append_log(f"Total
|
|
|
|
| 634 |
append_log("============================================")
|
| 635 |
|
| 636 |
-
|
| 637 |
-
|
| 638 |
-
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
|
| 642 |
-
sorted_bam, output_vcf = pl.generate_synthetic_bam(
|
| 643 |
-
work_dir=work_dir,
|
| 644 |
-
snvs_bed=snvs_bed,
|
| 645 |
-
fasta_path=fasta_path,
|
| 646 |
-
depth=depth,
|
| 647 |
-
vaf=vaf,
|
| 648 |
-
rg_id=rg_id,
|
| 649 |
-
rg_sm=rg_sm,
|
| 650 |
-
insert_size=insert_size,
|
| 651 |
-
insert_std=insert_std,
|
| 652 |
-
indel_interval=indel_interval,
|
| 653 |
-
read_length=read_length,
|
| 654 |
-
sequencing_mode="pcr_amplicon" if seq_mode.startswith("PCR Amplicon") else "hybrid_capture",
|
| 655 |
-
log_func=append_log,
|
| 656 |
-
progress_func=bam_progress,
|
| 657 |
-
compress_fastq=compress_fastq,
|
| 658 |
-
)
|
| 659 |
-
|
| 660 |
-
update_progress(1.0, "Done!")
|
| 661 |
-
append_log("\nβ
Pipeline complete.")
|
| 662 |
|
| 663 |
-
bai_path = Path(str(sorted_bam) + ".bai")
|
| 664 |
-
vcf_path = Path(output_vcf) if not isinstance(output_vcf, Path) else output_vcf
|
| 665 |
igv_bed_path = Path(snvs_bed) if not isinstance(snvs_bed, Path) else snvs_bed
|
| 666 |
fully_bed_path = Path(fully_bed) if fully_bed and not isinstance(fully_bed, Path) else fully_bed
|
|
|
|
| 667 |
|
| 668 |
-
|
| 669 |
-
|
| 670 |
-
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
|
| 674 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 675 |
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 679 |
if bai_path.exists():
|
| 680 |
-
shutil.
|
| 681 |
-
shutil.
|
| 682 |
-
shutil.copy(igv_bed_path, dest / "igv_variant_navigator.bed")
|
| 683 |
-
if fully_bed_path and fully_bed_path.exists():
|
| 684 |
-
shutil.copy(fully_bed_path, dest / "fully_covered_exons.bed")
|
| 685 |
-
if mane_transcripts_bed and mane_transcripts_bed.exists():
|
| 686 |
-
shutil.copy(mane_transcripts_bed, dest / "mane_transcripts.bed")
|
| 687 |
if fastq_r1_path.exists():
|
| 688 |
-
shutil.
|
| 689 |
if fastq_r2_path.exists():
|
| 690 |
-
shutil.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 691 |
|
| 692 |
# Store URLs and path strings β never load large files into session_state memory
|
| 693 |
st.session_state["results"] = {
|
| 694 |
-
"
|
| 695 |
-
"
|
| 696 |
-
"bam_path": str(sorted_bam),
|
| 697 |
-
"bai_path": str(bai_path) if bai_path.exists() else None,
|
| 698 |
-
"vcf_path": str(vcf_path),
|
| 699 |
-
"igv_bed_path": str(igv_bed_path),
|
| 700 |
-
"fully_covered_bed_path": str(fully_bed_path) if fully_bed_path else None,
|
| 701 |
-
"mane_transcripts_bed_path": str(mane_transcripts_bed) if mane_transcripts_bed else None,
|
| 702 |
-
"fastq_r1_path": str(fastq_r1_path) if fastq_r1_path.exists() else None,
|
| 703 |
-
"fastq_r2_path": str(fastq_r2_path) if fastq_r2_path.exists() else None,
|
| 704 |
-
"bam_url": f"/app/static/{work_dir_name}/synthetic.sorted.bam",
|
| 705 |
-
"bai_url": f"/app/static/{work_dir_name}/synthetic.sorted.bam.bai" if bai_path.exists() else None,
|
| 706 |
-
"vcf_url": f"/app/static/{work_dir_name}/synthetic.vcf",
|
| 707 |
-
"igv_bed_url": f"/app/static/{work_dir_name}/igv_variant_navigator.bed",
|
| 708 |
-
"fully_covered_bed_url": f"/app/static/{work_dir_name}/fully_covered_exons.bed" if fully_bed_path and fully_bed_path.exists() else None,
|
| 709 |
-
"mane_transcripts_bed_url": f"/app/static/{work_dir_name}/mane_transcripts.bed" if mane_transcripts_bed and mane_transcripts_bed.exists() else None,
|
| 710 |
-
"fastq_r1_url": f"/app/static/{work_dir_name}/{fastq_r1_path.name}" if fastq_r1_path.exists() else None,
|
| 711 |
-
"fastq_r2_url": f"/app/static/{work_dir_name}/{fastq_r2_path.name}" if fastq_r2_path.exists() else None,
|
| 712 |
-
"work_dir_name": work_dir_name,
|
| 713 |
-
"genome_version": genome_version,
|
| 714 |
-
"compress_fastq": compress_fastq,
|
| 715 |
}
|
| 716 |
st.session_state["log_lines"] = log_lines[:]
|
| 717 |
|
|
@@ -723,10 +851,8 @@ if run_btn:
|
|
| 723 |
# ββ Results section (persists across reruns via session_state) ββββββββββββββββ
|
| 724 |
|
| 725 |
if "results" in st.session_state:
|
| 726 |
-
|
| 727 |
-
|
| 728 |
-
total_snvs = res["total_snvs"]
|
| 729 |
-
|
| 730 |
st.success("Pipeline completed successfully!")
|
| 731 |
|
| 732 |
# Show log if available and pipeline didn't just run
|
|
@@ -735,6 +861,25 @@ if "results" in st.session_state:
|
|
| 735 |
st.code("\n".join(st.session_state["log_lines"][-80:]), language=None)
|
| 736 |
|
| 737 |
st.header("3 Β· Results")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 738 |
m1, m2, m3, m4 = st.columns(4)
|
| 739 |
m1.metric("Fully Covered Exons", f"{stats['fully_covered']:,}")
|
| 740 |
m2.metric("Partially Covered Exons", f"{stats['partially_covered']:,}")
|
|
|
|
| 254 |
with st.sidebar:
|
| 255 |
st.header("Pipeline Parameters")
|
| 256 |
|
| 257 |
+
enable_batch = st.checkbox(
|
| 258 |
+
"Enable Batch Controls Mode",
|
| 259 |
+
value=False,
|
| 260 |
+
help="Enable this to generate multiple in-silico controls (differing by variant type/VAF) at the exact same coordinates in a single run."
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
st.subheader("Sequencing Parameters")
|
| 264 |
depth = st.number_input("Target read depth per variant", min_value=1, max_value=10000, value=100, step=10)
|
| 265 |
+
if not enable_batch:
|
| 266 |
+
vaf = st.slider("Variant allele frequency (VAF)", min_value=0.01, max_value=1.0, value=0.20, step=0.01, format="%.2f")
|
| 267 |
+
else:
|
| 268 |
+
vaf = 0.20
|
| 269 |
read_length = st.number_input("Read length (bp)", min_value=50, max_value=300, value=150, step=10)
|
| 270 |
|
| 271 |
st.subheader("Sequencing Technology")
|
|
|
|
| 288 |
indel_interval = st.number_input(
|
| 289 |
"Indel interval (0 = SNVs only)",
|
| 290 |
min_value=0, max_value=100, value=10, step=1,
|
| 291 |
+
help="Make every Nth variant an indel. Set to 0 to generate only SNVs. Only active when using Mixed (Random) variants.",
|
| 292 |
)
|
| 293 |
|
| 294 |
+
st.subheader("Variant Specification")
|
| 295 |
+
if not enable_batch:
|
| 296 |
+
var_type = st.selectbox(
|
| 297 |
+
"Variant type",
|
| 298 |
+
options=["SNV", "Insertion", "Deletion", "Mixed (Random)"],
|
| 299 |
+
index=3,
|
| 300 |
+
help="Choose the type of variants to generate. Mixed (Random) alternates between SNVs and Indels."
|
| 301 |
+
)
|
| 302 |
+
if var_type in ["Insertion", "Deletion"]:
|
| 303 |
+
var_length = st.number_input(
|
| 304 |
+
"Indel length (bp)",
|
| 305 |
+
min_value=1, max_value=100, value=3, step=1,
|
| 306 |
+
help="The exact size of the insertion or deletion in base pairs."
|
| 307 |
+
)
|
| 308 |
+
else:
|
| 309 |
+
var_length = 0
|
| 310 |
+
else:
|
| 311 |
+
st.info("βΉοΈ Batch mode enabled. Variant specifications are configured in the batch table in the main panel.")
|
| 312 |
+
|
| 313 |
st.divider()
|
| 314 |
st.subheader("Reference Genome")
|
| 315 |
genome_version = st.selectbox(
|
|
|
|
| 507 |
|
| 508 |
total_selected = len(edited_df[edited_df["Select"] == True])
|
| 509 |
st.info(f"Selected {total_selected:,} of {len(df):,} probes ({total_selected/len(df)*100:.1f}%) for variant generation.")
|
| 510 |
+
if uploaded_bed and enable_batch:
|
| 511 |
+
st.header("1.8 Β· Configure Batch Runs")
|
| 512 |
+
st.caption("Define the list of controls to generate. All runs will share the same variant coordinates.")
|
| 513 |
+
|
| 514 |
+
import pandas as pd
|
| 515 |
+
default_batch = pd.DataFrame([
|
| 516 |
+
{"Control Name": "control_snv", "VAF": 0.20, "Variant Type": "SNV", "Indel Length": 0},
|
| 517 |
+
{"Control Name": "control_ins_3bp", "VAF": 0.20, "Variant Type": "Insertion", "Indel Length": 3},
|
| 518 |
+
{"Control Name": "control_del_5bp", "VAF": 0.20, "Variant Type": "Deletion", "Indel Length": 5},
|
| 519 |
+
])
|
| 520 |
+
|
| 521 |
+
if "batch_df" not in st.session_state:
|
| 522 |
+
st.session_state["batch_df"] = default_batch
|
| 523 |
+
|
| 524 |
+
batch_df = st.data_editor(
|
| 525 |
+
st.session_state["batch_df"],
|
| 526 |
+
num_rows="dynamic",
|
| 527 |
+
use_container_width=True,
|
| 528 |
+
column_config={
|
| 529 |
+
"Control Name": st.column_config.TextColumn("Control Name", required=True),
|
| 530 |
+
"VAF": st.column_config.NumberColumn("VAF", min_value=0.01, max_value=1.0, step=0.01, format="%.2f", required=True),
|
| 531 |
+
"Variant Type": st.column_config.SelectboxColumn("Variant Type", options=["SNV", "Insertion", "Deletion", "Mixed (Random)"], required=True),
|
| 532 |
+
"Indel Length": st.column_config.NumberColumn("Indel Length (bp)", min_value=0, max_value=100, step=1, default=0),
|
| 533 |
+
}
|
| 534 |
+
)
|
| 535 |
+
st.session_state["batch_df"] = batch_df
|
| 536 |
|
| 537 |
st.divider()
|
| 538 |
|
|
|
|
| 681 |
log_func=append_log
|
| 682 |
)
|
| 683 |
|
| 684 |
+
# Resolve runs list
|
| 685 |
+
if enable_batch:
|
| 686 |
+
runs_to_process = []
|
| 687 |
+
for _, row in batch_df.iterrows():
|
| 688 |
+
runs_to_process.append({
|
| 689 |
+
"name": row["Control Name"].strip(),
|
| 690 |
+
"vaf": float(row["VAF"]),
|
| 691 |
+
"type": "Mixed" if row["Variant Type"].startswith("Mixed") else row["Variant Type"],
|
| 692 |
+
"length": int(row["Indel Length"]),
|
| 693 |
+
})
|
| 694 |
+
else:
|
| 695 |
+
runs_to_process = [{
|
| 696 |
+
"name": "Single Control",
|
| 697 |
+
"vaf": vaf,
|
| 698 |
+
"type": "Mixed" if var_type.startswith("Mixed") else var_type,
|
| 699 |
+
"length": var_length,
|
| 700 |
+
}]
|
| 701 |
+
|
| 702 |
+
# Sanitize and deduplicate names
|
| 703 |
+
import re
|
| 704 |
+
seen_names = set()
|
| 705 |
+
sanitized_runs = []
|
| 706 |
+
for r in runs_to_process:
|
| 707 |
+
clean_name = re.sub(r'[^a-zA-Z0-9_]', '_', r["name"])
|
| 708 |
+
if not clean_name:
|
| 709 |
+
clean_name = "control"
|
| 710 |
+
original_clean = clean_name
|
| 711 |
+
counter = 1
|
| 712 |
+
while clean_name in seen_names:
|
| 713 |
+
clean_name = f"{original_clean}_{counter}"
|
| 714 |
+
counter += 1
|
| 715 |
+
seen_names.add(clean_name)
|
| 716 |
+
r["name"] = clean_name
|
| 717 |
+
sanitized_runs.append(r)
|
| 718 |
+
runs_to_process = sanitized_runs
|
| 719 |
+
|
| 720 |
append_log("\n============================================")
|
| 721 |
append_log(" VARIANT SUMMARY ")
|
| 722 |
append_log("============================================")
|
| 723 |
+
append_log(f"Total SNV loci generated: {total_snvs}")
|
| 724 |
+
append_log(f"Total runs to execute: {len(runs_to_process)}")
|
| 725 |
append_log("============================================")
|
| 726 |
|
| 727 |
+
runs_results = {}
|
| 728 |
+
total_runs = len(runs_to_process)
|
| 729 |
+
work_dir_name = work_dir.name
|
| 730 |
+
static_dest_cwd = Path("static") / work_dir_name
|
| 731 |
+
static_dest_script = Path(__file__).parent / "static" / work_dir_name
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 732 |
|
|
|
|
|
|
|
| 733 |
igv_bed_path = Path(snvs_bed) if not isinstance(snvs_bed, Path) else snvs_bed
|
| 734 |
fully_bed_path = Path(fully_bed) if fully_bed and not isinstance(fully_bed, Path) else fully_bed
|
| 735 |
+
mane_transcripts_bed_path = Path(mane_transcripts_bed) if mane_transcripts_bed and not isinstance(mane_transcripts_bed, Path) else mane_transcripts_bed
|
| 736 |
|
| 737 |
+
for run_idx, run in enumerate(runs_to_process):
|
| 738 |
+
run_name = run["name"]
|
| 739 |
+
run_vaf = run["vaf"]
|
| 740 |
+
run_type = run["type"]
|
| 741 |
+
run_length = run["length"]
|
| 742 |
+
|
| 743 |
+
append_log(f"\n=== Generating Synthetic BAM for {run_name} (VAF={run_vaf}, {run_type}) ===")
|
| 744 |
+
|
| 745 |
+
def bam_progress(fraction, label):
|
| 746 |
+
update_progress(0.40 + (run_idx + fraction) / total_runs * 0.55, f"[{run_name}] {label}")
|
| 747 |
+
|
| 748 |
+
sorted_bam, output_vcf = pl.generate_synthetic_bam(
|
| 749 |
+
work_dir=work_dir,
|
| 750 |
+
snvs_bed=snvs_bed,
|
| 751 |
+
fasta_path=fasta_path,
|
| 752 |
+
depth=depth,
|
| 753 |
+
vaf=run_vaf,
|
| 754 |
+
rg_id=rg_id,
|
| 755 |
+
rg_sm=rg_sm,
|
| 756 |
+
insert_size=insert_size,
|
| 757 |
+
insert_std=insert_std,
|
| 758 |
+
indel_interval=indel_interval,
|
| 759 |
+
read_length=read_length,
|
| 760 |
+
sequencing_mode="pcr_amplicon" if seq_mode.startswith("PCR Amplicon") else "hybrid_capture",
|
| 761 |
+
log_func=append_log,
|
| 762 |
+
progress_func=bam_progress,
|
| 763 |
+
compress_fastq=compress_fastq,
|
| 764 |
+
variant_type=run_type,
|
| 765 |
+
indel_length=run_length
|
| 766 |
+
)
|
| 767 |
|
| 768 |
+
# Resolve output paths in work_dir
|
| 769 |
+
bai_path = Path(str(sorted_bam) + ".bai")
|
| 770 |
+
vcf_path = Path(output_vcf) if not isinstance(output_vcf, Path) else output_vcf
|
| 771 |
+
fastq_r1_path = work_dir / ("synthetic_R1.fastq.gz" if compress_fastq else "synthetic_R1.fastq")
|
| 772 |
+
fastq_r2_path = work_dir / ("synthetic_R2.fastq.gz" if compress_fastq else "synthetic_R2.fastq")
|
| 773 |
+
|
| 774 |
+
# Relocate to run subfolder in work_dir
|
| 775 |
+
run_work_dir = work_dir / run_name
|
| 776 |
+
run_work_dir.mkdir(parents=True, exist_ok=True)
|
| 777 |
+
|
| 778 |
+
bam_reloc = run_work_dir / "synthetic.sorted.bam"
|
| 779 |
+
bai_reloc = run_work_dir / "synthetic.sorted.bam.bai"
|
| 780 |
+
vcf_reloc = run_work_dir / "synthetic.vcf"
|
| 781 |
+
fastq_r1_reloc = run_work_dir / fastq_r1_path.name
|
| 782 |
+
fastq_r2_reloc = run_work_dir / fastq_r2_path.name
|
| 783 |
+
|
| 784 |
+
shutil.move(str(sorted_bam), str(bam_reloc))
|
| 785 |
if bai_path.exists():
|
| 786 |
+
shutil.move(str(bai_path), str(bai_reloc))
|
| 787 |
+
shutil.move(str(vcf_path), str(vcf_reloc))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 788 |
if fastq_r1_path.exists():
|
| 789 |
+
shutil.move(str(fastq_r1_path), str(fastq_r1_reloc))
|
| 790 |
if fastq_r2_path.exists():
|
| 791 |
+
shutil.move(str(fastq_r2_path), str(fastq_r2_reloc))
|
| 792 |
+
|
| 793 |
+
# Copy to static subfolder dest / run_name
|
| 794 |
+
for dest_root in [static_dest_cwd, static_dest_script]:
|
| 795 |
+
dest = dest_root / run_name
|
| 796 |
+
dest.mkdir(parents=True, exist_ok=True)
|
| 797 |
+
|
| 798 |
+
shutil.copy(bam_reloc, dest / "synthetic.sorted.bam")
|
| 799 |
+
if bai_reloc.exists():
|
| 800 |
+
shutil.copy(bai_reloc, dest / "synthetic.sorted.bam.bai")
|
| 801 |
+
shutil.copy(vcf_reloc, dest / "synthetic.vcf")
|
| 802 |
+
shutil.copy(igv_bed_path, dest / "igv_variant_navigator.bed")
|
| 803 |
+
if fully_bed_path and fully_bed_path.exists():
|
| 804 |
+
shutil.copy(fully_bed_path, dest / "fully_covered_exons.bed")
|
| 805 |
+
if mane_transcripts_bed_path and mane_transcripts_bed_path.exists():
|
| 806 |
+
shutil.copy(mane_transcripts_bed_path, dest / "mane_transcripts.bed")
|
| 807 |
+
if fastq_r1_reloc.exists():
|
| 808 |
+
shutil.copy(fastq_r1_reloc, dest / fastq_r1_reloc.name)
|
| 809 |
+
if fastq_r2_reloc.exists():
|
| 810 |
+
shutil.copy(fastq_r2_reloc, dest / fastq_r2_reloc.name)
|
| 811 |
+
|
| 812 |
+
runs_results[run_name] = {
|
| 813 |
+
"stats": stats,
|
| 814 |
+
"total_snvs": total_snvs,
|
| 815 |
+
"bam_path": str(bam_reloc),
|
| 816 |
+
"bai_path": str(bai_reloc) if bai_reloc.exists() else None,
|
| 817 |
+
"vcf_path": str(vcf_reloc),
|
| 818 |
+
"igv_bed_path": str(igv_bed_path),
|
| 819 |
+
"fully_covered_bed_path": str(fully_bed_path) if fully_bed_path else None,
|
| 820 |
+
"mane_transcripts_bed_path": str(mane_transcripts_bed_path) if mane_transcripts_bed_path else None,
|
| 821 |
+
"fastq_r1_path": str(fastq_r1_reloc) if fastq_r1_reloc.exists() else None,
|
| 822 |
+
"fastq_r2_path": str(fastq_r2_reloc) if fastq_r2_reloc.exists() else None,
|
| 823 |
+
"bam_url": f"/app/static/{work_dir_name}/{run_name}/synthetic.sorted.bam",
|
| 824 |
+
"bai_url": f"/app/static/{work_dir_name}/{run_name}/synthetic.sorted.bam.bai" if bai_reloc.exists() else None,
|
| 825 |
+
"vcf_url": f"/app/static/{work_dir_name}/{run_name}/synthetic.vcf",
|
| 826 |
+
"igv_bed_url": f"/app/static/{work_dir_name}/{run_name}/igv_variant_navigator.bed",
|
| 827 |
+
"fully_covered_bed_url": f"/app/static/{work_dir_name}/{run_name}/fully_covered_exons.bed" if fully_bed_path and fully_bed_path.exists() else None,
|
| 828 |
+
"mane_transcripts_bed_url": f"/app/static/{work_dir_name}/{run_name}/mane_transcripts.bed" if mane_transcripts_bed_path and mane_transcripts_bed_path.exists() else None,
|
| 829 |
+
"fastq_r1_url": f"/app/static/{work_dir_name}/{run_name}/{fastq_r1_reloc.name}" if fastq_r1_reloc.exists() else None,
|
| 830 |
+
"fastq_r2_url": f"/app/static/{work_dir_name}/{run_name}/{fastq_r2_reloc.name}" if fastq_r2_reloc.exists() else None,
|
| 831 |
+
"work_dir_name": f"{work_dir_name}/{run_name}",
|
| 832 |
+
"genome_version": genome_version,
|
| 833 |
+
"compress_fastq": compress_fastq,
|
| 834 |
+
}
|
| 835 |
+
|
| 836 |
+
update_progress(1.0, "Done!")
|
| 837 |
+
append_log("\nβ
Pipeline complete.")
|
| 838 |
|
| 839 |
# Store URLs and path strings β never load large files into session_state memory
|
| 840 |
st.session_state["results"] = {
|
| 841 |
+
"is_batch": enable_batch,
|
| 842 |
+
"runs": runs_results
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 843 |
}
|
| 844 |
st.session_state["log_lines"] = log_lines[:]
|
| 845 |
|
|
|
|
| 851 |
# ββ Results section (persists across reruns via session_state) ββββββββββββββββ
|
| 852 |
|
| 853 |
if "results" in st.session_state:
|
| 854 |
+
raw_results = st.session_state["results"]
|
| 855 |
+
|
|
|
|
|
|
|
| 856 |
st.success("Pipeline completed successfully!")
|
| 857 |
|
| 858 |
# Show log if available and pipeline didn't just run
|
|
|
|
| 861 |
st.code("\n".join(st.session_state["log_lines"][-80:]), language=None)
|
| 862 |
|
| 863 |
st.header("3 Β· Results")
|
| 864 |
+
|
| 865 |
+
is_batch = raw_results.get("is_batch", False)
|
| 866 |
+
if is_batch:
|
| 867 |
+
run_names = list(raw_results["runs"].keys())
|
| 868 |
+
selected_run = st.selectbox(
|
| 869 |
+
"Select control run to view/download",
|
| 870 |
+
options=run_names,
|
| 871 |
+
help="Choose which batch control configuration to display and download outputs for."
|
| 872 |
+
)
|
| 873 |
+
res = raw_results["runs"][selected_run]
|
| 874 |
+
|
| 875 |
+
# Display run details
|
| 876 |
+
st.markdown(f"**Selected Run Parameters:** Variant Type: `{res.get('variant_type')}`, VAF: `{res.get('vaf')}`" +
|
| 877 |
+
(f", Indel Length: `{res.get('indel_length')} bp`" if res.get("variant_type") in ["Insertion", "Deletion"] else ""))
|
| 878 |
+
else:
|
| 879 |
+
res = list(raw_results["runs"].values())[0]
|
| 880 |
+
|
| 881 |
+
stats = res["stats"]
|
| 882 |
+
total_snvs = res["total_snvs"]
|
| 883 |
m1, m2, m3, m4 = st.columns(4)
|
| 884 |
m1.metric("Fully Covered Exons", f"{stats['fully_covered']:,}")
|
| 885 |
m2.metric("Partially Covered Exons", f"{stats['partially_covered']:,}")
|