Spaces:
Running on Zero
Running on Zero
segment length awareness
Browse files
app.py
CHANGED
|
@@ -64,6 +64,8 @@ COPY_SOURCE_AUDIO = os.getenv("SCAIL_COPY_SOURCE_AUDIO", "1") == "1"
|
|
| 64 |
MATCH_SOURCE_FPS_AT_EXPORT = os.getenv("SCAIL_MATCH_SOURCE_FPS_AT_EXPORT", "1") == "1"
|
| 65 |
CONFORM_OUTPUT_TO_SOURCE_FPS = os.getenv("SCAIL_CONFORM_OUTPUT_TO_SOURCE_FPS", "1") == "1"
|
| 66 |
FPS_MATCH_EPSILON = float(os.getenv("SCAIL_FPS_MATCH_EPSILON", "0.05"))
|
|
|
|
|
|
|
| 67 |
|
| 68 |
CLIP_CKPT_NAME = "models_clip_open-clip-xlm-roberta-large-vit-huge-14-onlyvisual.pth"
|
| 69 |
ORIGINAL_DIT_REL_PATH = "model/1/fsdp2_rank_0000_checkpoint.pt"
|
|
@@ -1001,6 +1003,50 @@ def _video_fps(video_path: str | Path) -> float | None:
|
|
| 1001 |
return None
|
| 1002 |
|
| 1003 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1004 |
def _generation_config_for_source_video(source_video_path: str | Path):
|
| 1005 |
cfg = copy.deepcopy(_SCAIL_CONFIGS[MODEL_NAME])
|
| 1006 |
if not MATCH_SOURCE_FPS_AT_EXPORT:
|
|
@@ -1189,6 +1235,7 @@ def _run_scail_job(
|
|
| 1189 |
pipeline = _get_pipeline()
|
| 1190 |
cfg = _generation_config_for_source_video(pose_path)
|
| 1191 |
save_file = OUTPUT_DIR / f"scail2_{uuid.uuid4().hex}.mp4"
|
|
|
|
| 1192 |
|
| 1193 |
if progress is not None:
|
| 1194 |
progress(0.12, desc="Preparing inputs")
|
|
@@ -1197,7 +1244,7 @@ def _run_scail_job(
|
|
| 1197 |
target_w=int(target_w),
|
| 1198 |
sample_shift=float(sample_shift),
|
| 1199 |
sample_solver=DEFAULT_SOLVER,
|
| 1200 |
-
segment_len=
|
| 1201 |
segment_overlap=int(segment_overlap),
|
| 1202 |
sample_steps=int(sample_steps),
|
| 1203 |
sample_guide_scale=float(guide_scale),
|
|
|
|
| 64 |
MATCH_SOURCE_FPS_AT_EXPORT = os.getenv("SCAIL_MATCH_SOURCE_FPS_AT_EXPORT", "1") == "1"
|
| 65 |
CONFORM_OUTPUT_TO_SOURCE_FPS = os.getenv("SCAIL_CONFORM_OUTPUT_TO_SOURCE_FPS", "1") == "1"
|
| 66 |
FPS_MATCH_EPSILON = float(os.getenv("SCAIL_FPS_MATCH_EPSILON", "0.05"))
|
| 67 |
+
AUTO_AVOID_SEGMENT_TRUNCATION = os.getenv("SCAIL_AUTO_AVOID_SEGMENT_TRUNCATION", "1") == "1"
|
| 68 |
+
MAX_AUTO_SEGMENT_LEN = int(os.getenv("SCAIL_MAX_AUTO_SEGMENT_LEN", "161"))
|
| 69 |
|
| 70 |
CLIP_CKPT_NAME = "models_clip_open-clip-xlm-roberta-large-vit-huge-14-onlyvisual.pth"
|
| 71 |
ORIGINAL_DIT_REL_PATH = "model/1/fsdp2_rank_0000_checkpoint.pt"
|
|
|
|
| 1003 |
return None
|
| 1004 |
|
| 1005 |
|
| 1006 |
+
def _video_frame_count(video_path: str | Path) -> int | None:
|
| 1007 |
+
try:
|
| 1008 |
+
import cv2
|
| 1009 |
+
|
| 1010 |
+
capture = cv2.VideoCapture(str(video_path))
|
| 1011 |
+
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
|
| 1012 |
+
capture.release()
|
| 1013 |
+
if frame_count > 0:
|
| 1014 |
+
return frame_count
|
| 1015 |
+
except Exception as exc:
|
| 1016 |
+
logging.warning("Could not read frame count for %s: %s", video_path, exc)
|
| 1017 |
+
return None
|
| 1018 |
+
|
| 1019 |
+
|
| 1020 |
+
def _effective_segment_len(requested_segment_len: int, segment_overlap: int, source_video_path: str | Path) -> int:
|
| 1021 |
+
requested_segment_len = int(requested_segment_len)
|
| 1022 |
+
segment_overlap = int(segment_overlap)
|
| 1023 |
+
if not AUTO_AVOID_SEGMENT_TRUNCATION:
|
| 1024 |
+
return requested_segment_len
|
| 1025 |
+
|
| 1026 |
+
frame_count = _video_frame_count(source_video_path)
|
| 1027 |
+
if frame_count is None or frame_count <= requested_segment_len:
|
| 1028 |
+
return requested_segment_len
|
| 1029 |
+
|
| 1030 |
+
if frame_count <= MAX_AUTO_SEGMENT_LEN:
|
| 1031 |
+
effective = max(frame_count, segment_overlap + 1)
|
| 1032 |
+
logging.info(
|
| 1033 |
+
"Increasing segment_len to avoid SCAIL-2 tail truncation: requested=%s frame_count=%s effective=%s",
|
| 1034 |
+
requested_segment_len,
|
| 1035 |
+
frame_count,
|
| 1036 |
+
effective,
|
| 1037 |
+
)
|
| 1038 |
+
return effective
|
| 1039 |
+
|
| 1040 |
+
logging.warning(
|
| 1041 |
+
"Source has %s frames, but segment_len=%s and max_auto_segment_len=%s. "
|
| 1042 |
+
"SCAIL-2 may drop the final partial segment unless you increase segment_len or trim/resample the input.",
|
| 1043 |
+
frame_count,
|
| 1044 |
+
requested_segment_len,
|
| 1045 |
+
MAX_AUTO_SEGMENT_LEN,
|
| 1046 |
+
)
|
| 1047 |
+
return requested_segment_len
|
| 1048 |
+
|
| 1049 |
+
|
| 1050 |
def _generation_config_for_source_video(source_video_path: str | Path):
|
| 1051 |
cfg = copy.deepcopy(_SCAIL_CONFIGS[MODEL_NAME])
|
| 1052 |
if not MATCH_SOURCE_FPS_AT_EXPORT:
|
|
|
|
| 1235 |
pipeline = _get_pipeline()
|
| 1236 |
cfg = _generation_config_for_source_video(pose_path)
|
| 1237 |
save_file = OUTPUT_DIR / f"scail2_{uuid.uuid4().hex}.mp4"
|
| 1238 |
+
effective_segment_len = _effective_segment_len(segment_len, segment_overlap, pose_path)
|
| 1239 |
|
| 1240 |
if progress is not None:
|
| 1241 |
progress(0.12, desc="Preparing inputs")
|
|
|
|
| 1244 |
target_w=int(target_w),
|
| 1245 |
sample_shift=float(sample_shift),
|
| 1246 |
sample_solver=DEFAULT_SOLVER,
|
| 1247 |
+
segment_len=effective_segment_len,
|
| 1248 |
segment_overlap=int(segment_overlap),
|
| 1249 |
sample_steps=int(sample_steps),
|
| 1250 |
sample_guide_scale=float(guide_scale),
|