Update app.py
Browse files
app.py
CHANGED
|
@@ -90,7 +90,15 @@ def escape_xml(text):
|
|
| 90 |
text = str(text)
|
| 91 |
return text.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """).replace("'", "'")
|
| 92 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
def timecode_to_frames(tc, fps):
|
|
|
|
| 94 |
if not tc or not re.match(r"\d{2}:\d{2}:\d{2}[:\.]\d{2}", tc): return 0
|
| 95 |
parts = re.split(r'[:\.]', tc)
|
| 96 |
h, m, s, f = map(int, parts)
|
|
@@ -248,7 +256,6 @@ def generate_xml(sequence_name, segments, clip_metadata, fps=25):
|
|
| 248 |
lines.append(f'\t\t\t\t\t<out>{src_out_frames}</out>')
|
| 249 |
lines.append(f'\t\t\t\t\t<masterclipid>{master_id_to_use}</masterclipid>')
|
| 250 |
|
| 251 |
-
# Define file only once per unique source
|
| 252 |
if master_id_to_use not in defined_files:
|
| 253 |
lines.append(f'\t\t\t\t\t<file id="{master_id_to_use}">')
|
| 254 |
lines.append(f'\t\t\t\t\t\t<name>{escape_xml(source_clip)}</name>')
|
|
@@ -263,7 +270,7 @@ def generate_xml(sequence_name, segments, clip_metadata, fps=25):
|
|
| 263 |
lines.append(f'\t\t\t\t\t\t</timecode>')
|
| 264 |
lines.append('\t\t\t\t\t\t<media>')
|
| 265 |
lines.append('\t\t\t\t\t\t\t<video><samplecharacteristics><width>1920</width><height>1080</height></samplecharacteristics></video>')
|
| 266 |
-
if not is_graphic:
|
| 267 |
lines.append('\t\t\t\t\t\t\t<audio><samplecharacteristics><depth>16</depth><samplerate>48000</samplerate></samplecharacteristics><channelcount>2</channelcount></audio>')
|
| 268 |
lines.append('\t\t\t\t\t\t</media>')
|
| 269 |
lines.append('\t\t\t\t\t</file>')
|
|
@@ -401,10 +408,9 @@ def call_gemini_for_edl(transcripts_dict, story_prompt, api_key, previous_edit=N
|
|
| 401 |
system_prompt = (
|
| 402 |
"You are an expert Documentary Senior Editor. Use the provided JSON object containing transcripts "
|
| 403 |
"from ONE OR MORE source clips to create a condensed story.\n\n"
|
| 404 |
-
"
|
| 405 |
-
"
|
| 406 |
-
"
|
| 407 |
-
"CRITICAL: ONLY generate 'vo' or 'graphic' segments if explicitly requested. Otherwise, use 'clip' segments.\n\n"
|
| 408 |
"Output ONLY a valid JSON array of segments. Every segment MUST be a 'clip', a 'vo', or a 'graphic'.\n\n"
|
| 409 |
"For 'clip' segments (extracting from the subject):\n"
|
| 410 |
"{\"type\": \"clip\", \"source_clip\": \"Cam_A_Interview\", \"src_start\": 12.5, \"src_end\": 25.0, \"note\": \"Subject talks about X\", \"gap\": 0.0}\n"
|
|
@@ -412,9 +418,9 @@ def call_gemini_for_edl(transcripts_dict, story_prompt, api_key, previous_edit=N
|
|
| 412 |
"- IGNORE ALL INTERVIEWER COMMENTS.\n"
|
| 413 |
"- REMOVE FLUFF: Delete 'um', 'ah', repeats.\n"
|
| 414 |
"- TIMESTAMP INTEGRITY: Use exact word-level start/end times.\n\n"
|
| 415 |
-
"For 'vo' segments:\n"
|
| 416 |
"{\"type\": \"vo\", \"text\": \"The journey began years ago...\", \"duration\": 3.5, \"gap\": 0.0}\n\n"
|
| 417 |
-
"For 'graphic' segments:\n"
|
| 418 |
"{\"type\": \"graphic\", \"text\": \"Chapter 1\", \"duration\": 4.0, \"gap\": 0.0}\n\n"
|
| 419 |
"PACING: Add a 'gap' (in seconds) between distinct ideas."
|
| 420 |
)
|
|
@@ -467,13 +473,19 @@ def call_gemini_for_edl(transcripts_dict, story_prompt, api_key, previous_edit=N
|
|
| 467 |
st.set_page_config(page_title="Junior Editor Pro", layout="wide")
|
| 468 |
st.title("Junior Editor (Multi-Clip Edition)")
|
| 469 |
|
| 470 |
-
st.
|
| 471 |
-
|
| 472 |
-
*
|
| 473 |
-
*
|
| 474 |
-
*
|
| 475 |
-
*
|
| 476 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 477 |
|
| 478 |
st.divider()
|
| 479 |
|
|
@@ -489,7 +501,6 @@ with st.sidebar:
|
|
| 489 |
st.header("Transcription Settings")
|
| 490 |
language_map = {"Auto-Detect": None, "English": "en", "Spanish": "es"}
|
| 491 |
target_language = language_map[st.selectbox("Audio Language", list(language_map.keys()), index=1)]
|
| 492 |
-
num_speakers = st.number_input("Speakers per clip (0=Auto)", min_value=0, value=0)
|
| 493 |
|
| 494 |
# -- Main Multi-Clip Bin area --
|
| 495 |
if "transcripts" not in st.session_state:
|
|
@@ -517,7 +528,8 @@ if uploaded_files and len(st.session_state.transcripts) == 0:
|
|
| 517 |
start_tc = st.text_input(
|
| 518 |
"Source Start Timecode",
|
| 519 |
value="00:00:00:00",
|
| 520 |
-
key=f"tc_{idx}"
|
|
|
|
| 521 |
)
|
| 522 |
clip_configs[f.name] = {"file": f, "custom_name": custom_name, "start_tc": start_tc}
|
| 523 |
|
|
@@ -535,7 +547,6 @@ if uploaded_files and len(st.session_state.transcripts) == 0:
|
|
| 535 |
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 536 |
compute_type = "float16" if device == "cuda" else "int8"
|
| 537 |
|
| 538 |
-
# Load heavy models ONCE outside the loop to save massive VRAM and time
|
| 539 |
status_text.markdown("**Loading AI Models into memory...**")
|
| 540 |
whisper_model = whisperx.load_model("large-v2", device, compute_type=compute_type)
|
| 541 |
diarize_model = whisperx.DiarizationPipeline(use_auth_token=ACTIVE_HF_TOKEN, device=device)
|
|
@@ -544,7 +555,7 @@ if uploaded_files and len(st.session_state.transcripts) == 0:
|
|
| 544 |
|
| 545 |
for i, (original_filename, config) in enumerate(clip_configs.items()):
|
| 546 |
c_name = config["custom_name"]
|
| 547 |
-
|
| 548 |
c_file = config["file"]
|
| 549 |
|
| 550 |
status_text.markdown(f"**Processing Clip {i+1}/{total_clips}: {c_name}...**")
|
|
@@ -561,8 +572,7 @@ if uploaded_files and len(st.session_state.transcripts) == 0:
|
|
| 561 |
gc.collect()
|
| 562 |
torch.cuda.empty_cache()
|
| 563 |
|
| 564 |
-
|
| 565 |
-
diarize_segments = diarize_model(audio, **diarize_kwargs)
|
| 566 |
final_result = whisperx.assign_word_speakers(diarize_segments, result)
|
| 567 |
|
| 568 |
processed_segments = []
|
|
@@ -574,10 +584,12 @@ if uploaded_files and len(st.session_state.transcripts) == 0:
|
|
| 574 |
"end": segment["end"]
|
| 575 |
})
|
| 576 |
|
|
|
|
|
|
|
| 577 |
all_transcripts[c_name] = processed_segments
|
| 578 |
all_metadata[c_name] = {
|
| 579 |
-
"start_tc":
|
| 580 |
-
"offset_frames": timecode_to_frames(
|
| 581 |
}
|
| 582 |
|
| 583 |
progress_bar.progress((i + 1) / total_clips)
|
|
@@ -608,7 +620,7 @@ if len(st.session_state.transcripts) > 0:
|
|
| 608 |
st.divider()
|
| 609 |
|
| 610 |
st.subheader("Your Instruction")
|
| 611 |
-
brief = st.text_area("What should the Junior Editor do?", placeholder="e.g. Build a 1-minute intro cutting between Cam_A and Cam_B.
|
| 612 |
|
| 613 |
if st.button("Generate Multi-Clip Edit", type="primary"):
|
| 614 |
with st.spinner("Analyzing all transcripts and building sequence..."):
|
|
|
|
| 90 |
text = str(text)
|
| 91 |
return text.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """).replace("'", "'")
|
| 92 |
|
| 93 |
+
def format_tc_string(tc):
|
| 94 |
+
"""Automatically adds colons to 8-digit timecodes (e.g., 01000000 -> 01:00:00:00)."""
|
| 95 |
+
tc_clean = re.sub(r'[^\d]', '', tc)
|
| 96 |
+
if len(tc_clean) >= 8:
|
| 97 |
+
return f"{tc_clean[0:2]}:{tc_clean[2:4]}:{tc_clean[4:6]}:{tc_clean[6:8]}"
|
| 98 |
+
return tc
|
| 99 |
+
|
| 100 |
def timecode_to_frames(tc, fps):
|
| 101 |
+
tc = format_tc_string(tc)
|
| 102 |
if not tc or not re.match(r"\d{2}:\d{2}:\d{2}[:\.]\d{2}", tc): return 0
|
| 103 |
parts = re.split(r'[:\.]', tc)
|
| 104 |
h, m, s, f = map(int, parts)
|
|
|
|
| 256 |
lines.append(f'\t\t\t\t\t<out>{src_out_frames}</out>')
|
| 257 |
lines.append(f'\t\t\t\t\t<masterclipid>{master_id_to_use}</masterclipid>')
|
| 258 |
|
|
|
|
| 259 |
if master_id_to_use not in defined_files:
|
| 260 |
lines.append(f'\t\t\t\t\t<file id="{master_id_to_use}">')
|
| 261 |
lines.append(f'\t\t\t\t\t\t<name>{escape_xml(source_clip)}</name>')
|
|
|
|
| 270 |
lines.append(f'\t\t\t\t\t\t</timecode>')
|
| 271 |
lines.append('\t\t\t\t\t\t<media>')
|
| 272 |
lines.append('\t\t\t\t\t\t\t<video><samplecharacteristics><width>1920</width><height>1080</height></samplecharacteristics></video>')
|
| 273 |
+
if not is_graphic:
|
| 274 |
lines.append('\t\t\t\t\t\t\t<audio><samplecharacteristics><depth>16</depth><samplerate>48000</samplerate></samplecharacteristics><channelcount>2</channelcount></audio>')
|
| 275 |
lines.append('\t\t\t\t\t\t</media>')
|
| 276 |
lines.append('\t\t\t\t\t</file>')
|
|
|
|
| 408 |
system_prompt = (
|
| 409 |
"You are an expert Documentary Senior Editor. Use the provided JSON object containing transcripts "
|
| 410 |
"from ONE OR MORE source clips to create a condensed story.\n\n"
|
| 411 |
+
"STRICT INSTRUCTIONS ON GENERATIVE ELEMENTS:\n"
|
| 412 |
+
"- DO NOT generate 'vo' (Voice-Over) or 'graphic' (Title Cards) segments UNLESS the user explicitly asks for them in their creative brief.\n"
|
| 413 |
+
"- By default, you must ONLY construct the edit using actual extracted 'clip' segments from the provided transcripts.\n\n"
|
|
|
|
| 414 |
"Output ONLY a valid JSON array of segments. Every segment MUST be a 'clip', a 'vo', or a 'graphic'.\n\n"
|
| 415 |
"For 'clip' segments (extracting from the subject):\n"
|
| 416 |
"{\"type\": \"clip\", \"source_clip\": \"Cam_A_Interview\", \"src_start\": 12.5, \"src_end\": 25.0, \"note\": \"Subject talks about X\", \"gap\": 0.0}\n"
|
|
|
|
| 418 |
"- IGNORE ALL INTERVIEWER COMMENTS.\n"
|
| 419 |
"- REMOVE FLUFF: Delete 'um', 'ah', repeats.\n"
|
| 420 |
"- TIMESTAMP INTEGRITY: Use exact word-level start/end times.\n\n"
|
| 421 |
+
"For 'vo' segments (ONLY IF EXPLICITLY REQUESTED):\n"
|
| 422 |
"{\"type\": \"vo\", \"text\": \"The journey began years ago...\", \"duration\": 3.5, \"gap\": 0.0}\n\n"
|
| 423 |
+
"For 'graphic' segments (ONLY IF EXPLICITLY REQUESTED):\n"
|
| 424 |
"{\"type\": \"graphic\", \"text\": \"Chapter 1\", \"duration\": 4.0, \"gap\": 0.0}\n\n"
|
| 425 |
"PACING: Add a 'gap' (in seconds) between distinct ideas."
|
| 426 |
)
|
|
|
|
| 473 |
st.set_page_config(page_title="Junior Editor Pro", layout="wide")
|
| 474 |
st.title("Junior Editor (Multi-Clip Edition)")
|
| 475 |
|
| 476 |
+
with st.expander("Instructions - Please Read First", expanded=False):
|
| 477 |
+
st.markdown("""
|
| 478 |
+
* This program will generate an EDL or XML containing multiple interview cut down and interwoven as directed by you.
|
| 479 |
+
* Set your export timeline FPS and EDL or XML. If you are doing sequences it will have to be XML, but this will only work in Resolve (I haven’t tried FCP). If you are exporting sequences have the TC start at 00:00:00:00 for speed later.
|
| 480 |
+
* First, export compressed source files from your timeline. These can be sequences (inc multicam) or individual clips.
|
| 481 |
+
* Drag and drop multiple compressed files below (video or audio). AAC MP4’s work well for speed of upload.
|
| 482 |
+
* Enter the exact name of each your sequence or clip as well as the start timecode of the reference. You can add TC’s as `01000000` and the program will add colons for you.
|
| 483 |
+
* Junior Editor will transcribe and separate speakers. This takes approx 10 mins per hour of audio. Once completed.
|
| 484 |
+
* You can then instruct it to find engaging bits or construct a narrative. You can ask it to add in suggested VO or Graphics cards if you are planning to use voice over.
|
| 485 |
+
* It will create an EDL or XML to import back into Resolve.
|
| 486 |
+
* If you don’t like the first attempt, give it new instructions and ask it to generate a new sequence.
|
| 487 |
+
* Resolve Users: Uncheck "Automatically import source clips into media pool" during import.
|
| 488 |
+
""")
|
| 489 |
|
| 490 |
st.divider()
|
| 491 |
|
|
|
|
| 501 |
st.header("Transcription Settings")
|
| 502 |
language_map = {"Auto-Detect": None, "English": "en", "Spanish": "es"}
|
| 503 |
target_language = language_map[st.selectbox("Audio Language", list(language_map.keys()), index=1)]
|
|
|
|
| 504 |
|
| 505 |
# -- Main Multi-Clip Bin area --
|
| 506 |
if "transcripts" not in st.session_state:
|
|
|
|
| 528 |
start_tc = st.text_input(
|
| 529 |
"Source Start Timecode",
|
| 530 |
value="00:00:00:00",
|
| 531 |
+
key=f"tc_{idx}",
|
| 532 |
+
help="Format HH:MM:SS:FF or HHMMSSFF. We will auto-format it."
|
| 533 |
)
|
| 534 |
clip_configs[f.name] = {"file": f, "custom_name": custom_name, "start_tc": start_tc}
|
| 535 |
|
|
|
|
| 547 |
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 548 |
compute_type = "float16" if device == "cuda" else "int8"
|
| 549 |
|
|
|
|
| 550 |
status_text.markdown("**Loading AI Models into memory...**")
|
| 551 |
whisper_model = whisperx.load_model("large-v2", device, compute_type=compute_type)
|
| 552 |
diarize_model = whisperx.DiarizationPipeline(use_auth_token=ACTIVE_HF_TOKEN, device=device)
|
|
|
|
| 555 |
|
| 556 |
for i, (original_filename, config) in enumerate(clip_configs.items()):
|
| 557 |
c_name = config["custom_name"]
|
| 558 |
+
raw_tc = config["start_tc"]
|
| 559 |
c_file = config["file"]
|
| 560 |
|
| 561 |
status_text.markdown(f"**Processing Clip {i+1}/{total_clips}: {c_name}...**")
|
|
|
|
| 572 |
gc.collect()
|
| 573 |
torch.cuda.empty_cache()
|
| 574 |
|
| 575 |
+
diarize_segments = diarize_model(audio) # Removed min/max speaker restrictions
|
|
|
|
| 576 |
final_result = whisperx.assign_word_speakers(diarize_segments, result)
|
| 577 |
|
| 578 |
processed_segments = []
|
|
|
|
| 584 |
"end": segment["end"]
|
| 585 |
})
|
| 586 |
|
| 587 |
+
formatted_tc = format_tc_string(raw_tc)
|
| 588 |
+
|
| 589 |
all_transcripts[c_name] = processed_segments
|
| 590 |
all_metadata[c_name] = {
|
| 591 |
+
"start_tc": formatted_tc,
|
| 592 |
+
"offset_frames": timecode_to_frames(formatted_tc, fps)
|
| 593 |
}
|
| 594 |
|
| 595 |
progress_bar.progress((i + 1) / total_clips)
|
|
|
|
| 620 |
st.divider()
|
| 621 |
|
| 622 |
st.subheader("Your Instruction")
|
| 623 |
+
brief = st.text_area("What should the Junior Editor do?", placeholder="e.g. Build a 1-minute intro cutting between Cam_A and Cam_B.")
|
| 624 |
|
| 625 |
if st.button("Generate Multi-Clip Edit", type="primary"):
|
| 626 |
with st.spinner("Analyzing all transcripts and building sequence..."):
|