Upload 31 files
Browse files- .dockerignore +30 -0
- .gitignore +435 -427
- .python-version +1 -0
- Dockerfile +26 -0
- LICENSE +674 -674
- MANIFEST.in +9 -0
- app.py +436 -110
- debug_cpu.py +40 -0
- device_utils.py +506 -0
- docker-compose.yml +17 -0
- download_utils.py +407 -0
- ffmpeg_utils.py +169 -165
- file_utils.py +170 -170
- gemini_utils.py +840 -0
- install.bat +23 -0
- install_gpu.bat +7 -0
- legen-beta.ipynb +345 -0
- legen.ipynb +337 -227
- legen.py +742 -280
- legen_cli.py +57 -0
- pyproject.toml +76 -40
- pytest.ini +5 -0
- requirements.txt +13 -11
- start.bat +13 -0
- subtitle_utils.py +226 -171
- translate_utils.py +438 -232
- utils.py +102 -72
- uv.lock +0 -0
- whisper_utils.py +61 -50
- whisperx_utils.py +87 -79
.dockerignore
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
*.pyo
|
| 4 |
+
*.pyd
|
| 5 |
+
*.so
|
| 6 |
+
*.dylib
|
| 7 |
+
*.egg-info/
|
| 8 |
+
.eggs/
|
| 9 |
+
.Python
|
| 10 |
+
build/
|
| 11 |
+
dist/
|
| 12 |
+
venv/
|
| 13 |
+
.env
|
| 14 |
+
.env.local
|
| 15 |
+
.venv
|
| 16 |
+
pip-wheel-metadata/
|
| 17 |
+
.git
|
| 18 |
+
.gitignore
|
| 19 |
+
.idea/
|
| 20 |
+
.vscode/
|
| 21 |
+
.DS_Store
|
| 22 |
+
__MACOSX/
|
| 23 |
+
# Temporary pipelines
|
| 24 |
+
*.log
|
| 25 |
+
temp/
|
| 26 |
+
tmp/
|
| 27 |
+
softsubs_m/
|
| 28 |
+
hardsubs_m/
|
| 29 |
+
downloads/
|
| 30 |
+
data/
|
.gitignore
CHANGED
|
@@ -1,427 +1,435 @@
|
|
| 1 |
-
## Ignore Visual Studio temporary files, build results, and
|
| 2 |
-
## files generated by popular Visual Studio add-ons.
|
| 3 |
-
##
|
| 4 |
-
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
|
| 5 |
-
|
| 6 |
-
# User-specific files
|
| 7 |
-
*.rsuser
|
| 8 |
-
*.suo
|
| 9 |
-
*.user
|
| 10 |
-
*.userosscache
|
| 11 |
-
*.sln.docstates
|
| 12 |
-
|
| 13 |
-
# User-specific files (MonoDevelop/Xamarin Studio)
|
| 14 |
-
*.userprefs
|
| 15 |
-
|
| 16 |
-
# Mono auto generated files
|
| 17 |
-
mono_crash.*
|
| 18 |
-
|
| 19 |
-
# Build results
|
| 20 |
-
[Dd]ebug/
|
| 21 |
-
[Dd]ebugPublic/
|
| 22 |
-
[Rr]elease/
|
| 23 |
-
[Rr]eleases/
|
| 24 |
-
x64/
|
| 25 |
-
x86/
|
| 26 |
-
[Aa][Rr][Mm]/
|
| 27 |
-
[Aa][Rr][Mm]64/
|
| 28 |
-
bld/
|
| 29 |
-
[Bb]in/
|
| 30 |
-
[Oo]bj/
|
| 31 |
-
[Ll]og/
|
| 32 |
-
[Ll]ogs/
|
| 33 |
-
|
| 34 |
-
# Visual Studio 2015/2017 cache/options directory
|
| 35 |
-
.vs/
|
| 36 |
-
# Uncomment if you have tasks that create the project's static files in wwwroot
|
| 37 |
-
#wwwroot/
|
| 38 |
-
|
| 39 |
-
# Visual Studio 2017 auto generated files
|
| 40 |
-
Generated\ Files/
|
| 41 |
-
|
| 42 |
-
# MSTest test Results
|
| 43 |
-
[Tt]est[Rr]esult*/
|
| 44 |
-
[Bb]uild[Ll]og.*
|
| 45 |
-
|
| 46 |
-
# NUnit
|
| 47 |
-
*.VisualState.xml
|
| 48 |
-
TestResult.xml
|
| 49 |
-
nunit-*.xml
|
| 50 |
-
|
| 51 |
-
# Build Results of an ATL Project
|
| 52 |
-
[Dd]ebugPS/
|
| 53 |
-
[Rr]eleasePS/
|
| 54 |
-
dlldata.c
|
| 55 |
-
|
| 56 |
-
# Benchmark Results
|
| 57 |
-
BenchmarkDotNet.Artifacts/
|
| 58 |
-
|
| 59 |
-
# .NET Core
|
| 60 |
-
project.lock.json
|
| 61 |
-
project.fragment.lock.json
|
| 62 |
-
artifacts/
|
| 63 |
-
|
| 64 |
-
# StyleCop
|
| 65 |
-
StyleCopReport.xml
|
| 66 |
-
|
| 67 |
-
# Files built by Visual Studio
|
| 68 |
-
*_i.c
|
| 69 |
-
*_p.c
|
| 70 |
-
*_h.h
|
| 71 |
-
*.ilk
|
| 72 |
-
*.meta
|
| 73 |
-
*.obj
|
| 74 |
-
*.iobj
|
| 75 |
-
*.pch
|
| 76 |
-
*.pdb
|
| 77 |
-
*.ipdb
|
| 78 |
-
*.pgc
|
| 79 |
-
*.pgd
|
| 80 |
-
*.rsp
|
| 81 |
-
*.sbr
|
| 82 |
-
*.tlb
|
| 83 |
-
*.tli
|
| 84 |
-
*.tlh
|
| 85 |
-
*.tmp
|
| 86 |
-
*.tmp_proj
|
| 87 |
-
*_wpftmp.csproj
|
| 88 |
-
*.log
|
| 89 |
-
*.vspscc
|
| 90 |
-
*.vssscc
|
| 91 |
-
.builds
|
| 92 |
-
*.pidb
|
| 93 |
-
*.svclog
|
| 94 |
-
*.scc
|
| 95 |
-
|
| 96 |
-
# Chutzpah Test files
|
| 97 |
-
_Chutzpah*
|
| 98 |
-
|
| 99 |
-
# Visual C++ cache files
|
| 100 |
-
ipch/
|
| 101 |
-
*.aps
|
| 102 |
-
*.ncb
|
| 103 |
-
*.opendb
|
| 104 |
-
*.opensdf
|
| 105 |
-
*.sdf
|
| 106 |
-
*.cachefile
|
| 107 |
-
*.VC.db
|
| 108 |
-
*.VC.VC.opendb
|
| 109 |
-
|
| 110 |
-
# Visual Studio profiler
|
| 111 |
-
*.psess
|
| 112 |
-
*.vsp
|
| 113 |
-
*.vspx
|
| 114 |
-
*.sap
|
| 115 |
-
|
| 116 |
-
# Visual Studio Trace Files
|
| 117 |
-
*.e2e
|
| 118 |
-
|
| 119 |
-
# TFS 2012 Local Workspace
|
| 120 |
-
$tf/
|
| 121 |
-
|
| 122 |
-
# Guidance Automation Toolkit
|
| 123 |
-
*.gpState
|
| 124 |
-
|
| 125 |
-
# ReSharper is a .NET coding add-in
|
| 126 |
-
_ReSharper*/
|
| 127 |
-
*.[Rr]e[Ss]harper
|
| 128 |
-
*.DotSettings.user
|
| 129 |
-
|
| 130 |
-
# TeamCity is a build add-in
|
| 131 |
-
_TeamCity*
|
| 132 |
-
|
| 133 |
-
# DotCover is a Code Coverage Tool
|
| 134 |
-
*.dotCover
|
| 135 |
-
|
| 136 |
-
# AxoCover is a Code Coverage Tool
|
| 137 |
-
.axoCover/*
|
| 138 |
-
!.axoCover/settings.json
|
| 139 |
-
|
| 140 |
-
# Visual Studio code coverage results
|
| 141 |
-
*.coverage
|
| 142 |
-
*.coveragexml
|
| 143 |
-
|
| 144 |
-
# NCrunch
|
| 145 |
-
_NCrunch_*
|
| 146 |
-
.*crunch*.local.xml
|
| 147 |
-
nCrunchTemp_*
|
| 148 |
-
|
| 149 |
-
# MightyMoose
|
| 150 |
-
*.mm.*
|
| 151 |
-
AutoTest.Net/
|
| 152 |
-
|
| 153 |
-
# Web workbench (sass)
|
| 154 |
-
.sass-cache/
|
| 155 |
-
|
| 156 |
-
# Installshield output folder
|
| 157 |
-
[Ee]xpress/
|
| 158 |
-
|
| 159 |
-
# DocProject is a documentation generator add-in
|
| 160 |
-
DocProject/buildhelp/
|
| 161 |
-
DocProject/Help/*.HxT
|
| 162 |
-
DocProject/Help/*.HxC
|
| 163 |
-
DocProject/Help/*.hhc
|
| 164 |
-
DocProject/Help/*.hhk
|
| 165 |
-
DocProject/Help/*.hhp
|
| 166 |
-
DocProject/Help/Html2
|
| 167 |
-
DocProject/Help/html
|
| 168 |
-
|
| 169 |
-
# Click-Once directory
|
| 170 |
-
publish/
|
| 171 |
-
|
| 172 |
-
# Publish Web Output
|
| 173 |
-
*.[Pp]ublish.xml
|
| 174 |
-
*.azurePubxml
|
| 175 |
-
# Note: Comment the next line if you want to checkin your web deploy settings,
|
| 176 |
-
# but database connection strings (with potential passwords) will be unencrypted
|
| 177 |
-
*.pubxml
|
| 178 |
-
*.publishproj
|
| 179 |
-
|
| 180 |
-
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
| 181 |
-
# checkin your Azure Web App publish settings, but sensitive information contained
|
| 182 |
-
# in these scripts will be unencrypted
|
| 183 |
-
PublishScripts/
|
| 184 |
-
|
| 185 |
-
# NuGet Packages
|
| 186 |
-
*.nupkg
|
| 187 |
-
# NuGet Symbol Packages
|
| 188 |
-
*.snupkg
|
| 189 |
-
# The packages folder can be ignored because of Package Restore
|
| 190 |
-
**/[Pp]ackages/*
|
| 191 |
-
# except build/, which is used as an MSBuild target.
|
| 192 |
-
!**/[Pp]ackages/build/
|
| 193 |
-
# Uncomment if necessary however generally it will be regenerated when needed
|
| 194 |
-
#!**/[Pp]ackages/repositories.config
|
| 195 |
-
# NuGet v3's project.json files produces more ignorable files
|
| 196 |
-
*.nuget.props
|
| 197 |
-
*.nuget.targets
|
| 198 |
-
|
| 199 |
-
# Microsoft Azure Build Output
|
| 200 |
-
csx/
|
| 201 |
-
*.build.csdef
|
| 202 |
-
|
| 203 |
-
# Microsoft Azure Emulator
|
| 204 |
-
ecf/
|
| 205 |
-
rcf/
|
| 206 |
-
|
| 207 |
-
# Windows Store app package directories and files
|
| 208 |
-
AppPackages/
|
| 209 |
-
BundleArtifacts/
|
| 210 |
-
Package.StoreAssociation.xml
|
| 211 |
-
_pkginfo.txt
|
| 212 |
-
*.appx
|
| 213 |
-
*.appxbundle
|
| 214 |
-
*.appxupload
|
| 215 |
-
|
| 216 |
-
# Visual Studio cache files
|
| 217 |
-
# files ending in .cache can be ignored
|
| 218 |
-
*.[Cc]ache
|
| 219 |
-
# but keep track of directories ending in .cache
|
| 220 |
-
!?*.[Cc]ache/
|
| 221 |
-
|
| 222 |
-
# Others
|
| 223 |
-
ClientBin/
|
| 224 |
-
~$*
|
| 225 |
-
*~
|
| 226 |
-
*.dbmdl
|
| 227 |
-
*.dbproj.schemaview
|
| 228 |
-
*.jfm
|
| 229 |
-
*.pfx
|
| 230 |
-
*.publishsettings
|
| 231 |
-
orleans.codegen.cs
|
| 232 |
-
|
| 233 |
-
# Including strong name files can present a security risk
|
| 234 |
-
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
|
| 235 |
-
#*.snk
|
| 236 |
-
|
| 237 |
-
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
| 238 |
-
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
| 239 |
-
#bower_components/
|
| 240 |
-
|
| 241 |
-
# RIA/Silverlight projects
|
| 242 |
-
Generated_Code/
|
| 243 |
-
|
| 244 |
-
# Backup & report files from converting an old project file
|
| 245 |
-
# to a newer Visual Studio version. Backup files are not needed,
|
| 246 |
-
# because we have git ;-)
|
| 247 |
-
_UpgradeReport_Files/
|
| 248 |
-
Backup*/
|
| 249 |
-
UpgradeLog*.XML
|
| 250 |
-
UpgradeLog*.htm
|
| 251 |
-
ServiceFabricBackup/
|
| 252 |
-
*.rptproj.bak
|
| 253 |
-
|
| 254 |
-
# SQL Server files
|
| 255 |
-
*.mdf
|
| 256 |
-
*.ldf
|
| 257 |
-
*.ndf
|
| 258 |
-
|
| 259 |
-
# Business Intelligence projects
|
| 260 |
-
*.rdl.data
|
| 261 |
-
*.bim.layout
|
| 262 |
-
*.bim_*.settings
|
| 263 |
-
*.rptproj.rsuser
|
| 264 |
-
*- [Bb]ackup.rdl
|
| 265 |
-
*- [Bb]ackup ([0-9]).rdl
|
| 266 |
-
*- [Bb]ackup ([0-9][0-9]).rdl
|
| 267 |
-
|
| 268 |
-
# Microsoft Fakes
|
| 269 |
-
FakesAssemblies/
|
| 270 |
-
|
| 271 |
-
# GhostDoc plugin setting file
|
| 272 |
-
*.GhostDoc.xml
|
| 273 |
-
|
| 274 |
-
# Node.js Tools for Visual Studio
|
| 275 |
-
.ntvs_analysis.dat
|
| 276 |
-
node_modules/
|
| 277 |
-
|
| 278 |
-
# Visual Studio 6 build log
|
| 279 |
-
*.plg
|
| 280 |
-
|
| 281 |
-
# Visual Studio 6 workspace options file
|
| 282 |
-
*.opt
|
| 283 |
-
|
| 284 |
-
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
|
| 285 |
-
*.vbw
|
| 286 |
-
|
| 287 |
-
# Visual Studio LightSwitch build output
|
| 288 |
-
**/*.HTMLClient/GeneratedArtifacts
|
| 289 |
-
**/*.DesktopClient/GeneratedArtifacts
|
| 290 |
-
**/*.DesktopClient/ModelManifest.xml
|
| 291 |
-
**/*.Server/GeneratedArtifacts
|
| 292 |
-
**/*.Server/ModelManifest.xml
|
| 293 |
-
_Pvt_Extensions
|
| 294 |
-
|
| 295 |
-
# Paket dependency manager
|
| 296 |
-
.paket/paket.exe
|
| 297 |
-
paket-files/
|
| 298 |
-
|
| 299 |
-
# FAKE - F# Make
|
| 300 |
-
.fake/
|
| 301 |
-
|
| 302 |
-
# CodeRush personal settings
|
| 303 |
-
.cr/personal
|
| 304 |
-
|
| 305 |
-
# Python Tools for Visual Studio (PTVS)
|
| 306 |
-
__pycache__/
|
| 307 |
-
*.pyc
|
| 308 |
-
|
| 309 |
-
# Cake - Uncomment if you are using it
|
| 310 |
-
# tools/**
|
| 311 |
-
# !tools/packages.config
|
| 312 |
-
|
| 313 |
-
# Tabs Studio
|
| 314 |
-
*.tss
|
| 315 |
-
|
| 316 |
-
# Telerik's JustMock configuration file
|
| 317 |
-
*.jmconfig
|
| 318 |
-
|
| 319 |
-
# BizTalk build output
|
| 320 |
-
*.btp.cs
|
| 321 |
-
*.btm.cs
|
| 322 |
-
*.odx.cs
|
| 323 |
-
*.xsd.cs
|
| 324 |
-
|
| 325 |
-
# OpenCover UI analysis results
|
| 326 |
-
OpenCover/
|
| 327 |
-
|
| 328 |
-
# Azure Stream Analytics local run output
|
| 329 |
-
ASALocalRun/
|
| 330 |
-
|
| 331 |
-
# MSBuild Binary and Structured Log
|
| 332 |
-
*.binlog
|
| 333 |
-
|
| 334 |
-
# NVidia Nsight GPU debugger configuration file
|
| 335 |
-
*.nvuser
|
| 336 |
-
|
| 337 |
-
# MFractors (Xamarin productivity tool) working folder
|
| 338 |
-
.mfractor/
|
| 339 |
-
|
| 340 |
-
# Local History for Visual Studio
|
| 341 |
-
.localhistory/
|
| 342 |
-
|
| 343 |
-
# BeatPulse healthcheck temp database
|
| 344 |
-
healthchecksdb
|
| 345 |
-
|
| 346 |
-
# Backup folder for Package Reference Convert tool in Visual Studio 2017
|
| 347 |
-
MigrationBackup/
|
| 348 |
-
|
| 349 |
-
# Ionide (cross platform F# VS Code tools) working folder
|
| 350 |
-
.ionide/
|
| 351 |
-
|
| 352 |
-
vidqa_data/
|
| 353 |
-
media/
|
| 354 |
-
temp/
|
| 355 |
-
legen_burned_*/
|
| 356 |
-
legen_srt_*/
|
| 357 |
-
log-vidqa.txt
|
| 358 |
-
legen_errors.txt
|
| 359 |
-
legen-errors.txt
|
| 360 |
-
.vscode/launch.json
|
| 361 |
-
|
| 362 |
-
**.srt
|
| 363 |
-
**.mp4
|
| 364 |
-
**.webm
|
| 365 |
-
**.mkv
|
| 366 |
-
**.avi
|
| 367 |
-
**.mov
|
| 368 |
-
**.wmv
|
| 369 |
-
**.flv
|
| 370 |
-
**.vob
|
| 371 |
-
**.mts
|
| 372 |
-
**.m2ts
|
| 373 |
-
**.ts
|
| 374 |
-
**.yuv
|
| 375 |
-
**.mpg
|
| 376 |
-
**.mpga
|
| 377 |
-
**.mp2
|
| 378 |
-
**.mpeg
|
| 379 |
-
**.mpe
|
| 380 |
-
**.mpv
|
| 381 |
-
**.m2v
|
| 382 |
-
**.m4v
|
| 383 |
-
**.3gp
|
| 384 |
-
**.3g2
|
| 385 |
-
**.nsv
|
| 386 |
-
**.aa
|
| 387 |
-
**.aac
|
| 388 |
-
**.aax
|
| 389 |
-
**.act
|
| 390 |
-
**.aiff
|
| 391 |
-
**.alac
|
| 392 |
-
**.amr
|
| 393 |
-
**.ape
|
| 394 |
-
**.au
|
| 395 |
-
**.awb
|
| 396 |
-
**.dss
|
| 397 |
-
**.dvf
|
| 398 |
-
**.flac
|
| 399 |
-
**.gsm
|
| 400 |
-
**.iklax
|
| 401 |
-
**.ivs
|
| 402 |
-
**.m4a
|
| 403 |
-
**.m4b
|
| 404 |
-
**.m4p
|
| 405 |
-
**.mmf
|
| 406 |
-
**.mp3
|
| 407 |
-
**.mpc
|
| 408 |
-
**.msv
|
| 409 |
-
**.nmf
|
| 410 |
-
**.ogg
|
| 411 |
-
**.oga
|
| 412 |
-
**.mogg
|
| 413 |
-
**.opus
|
| 414 |
-
**.ra
|
| 415 |
-
**.rm
|
| 416 |
-
**.raw
|
| 417 |
-
**.rf64
|
| 418 |
-
**.sln
|
| 419 |
-
**.tta
|
| 420 |
-
**.voc
|
| 421 |
-
**.vox
|
| 422 |
-
**.wav
|
| 423 |
-
**.wma
|
| 424 |
-
**.wv
|
| 425 |
-
**.webm
|
| 426 |
-
**.8svx
|
| 427 |
-
**.mts
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## Ignore Visual Studio temporary files, build results, and
|
| 2 |
+
## files generated by popular Visual Studio add-ons.
|
| 3 |
+
##
|
| 4 |
+
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
|
| 5 |
+
|
| 6 |
+
# User-specific files
|
| 7 |
+
*.rsuser
|
| 8 |
+
*.suo
|
| 9 |
+
*.user
|
| 10 |
+
*.userosscache
|
| 11 |
+
*.sln.docstates
|
| 12 |
+
|
| 13 |
+
# User-specific files (MonoDevelop/Xamarin Studio)
|
| 14 |
+
*.userprefs
|
| 15 |
+
|
| 16 |
+
# Mono auto generated files
|
| 17 |
+
mono_crash.*
|
| 18 |
+
|
| 19 |
+
# Build results
|
| 20 |
+
[Dd]ebug/
|
| 21 |
+
[Dd]ebugPublic/
|
| 22 |
+
[Rr]elease/
|
| 23 |
+
[Rr]eleases/
|
| 24 |
+
x64/
|
| 25 |
+
x86/
|
| 26 |
+
[Aa][Rr][Mm]/
|
| 27 |
+
[Aa][Rr][Mm]64/
|
| 28 |
+
bld/
|
| 29 |
+
[Bb]in/
|
| 30 |
+
[Oo]bj/
|
| 31 |
+
[Ll]og/
|
| 32 |
+
[Ll]ogs/
|
| 33 |
+
|
| 34 |
+
# Visual Studio 2015/2017 cache/options directory
|
| 35 |
+
.vs/
|
| 36 |
+
# Uncomment if you have tasks that create the project's static files in wwwroot
|
| 37 |
+
#wwwroot/
|
| 38 |
+
|
| 39 |
+
# Visual Studio 2017 auto generated files
|
| 40 |
+
Generated\ Files/
|
| 41 |
+
|
| 42 |
+
# MSTest test Results
|
| 43 |
+
[Tt]est[Rr]esult*/
|
| 44 |
+
[Bb]uild[Ll]og.*
|
| 45 |
+
|
| 46 |
+
# NUnit
|
| 47 |
+
*.VisualState.xml
|
| 48 |
+
TestResult.xml
|
| 49 |
+
nunit-*.xml
|
| 50 |
+
|
| 51 |
+
# Build Results of an ATL Project
|
| 52 |
+
[Dd]ebugPS/
|
| 53 |
+
[Rr]eleasePS/
|
| 54 |
+
dlldata.c
|
| 55 |
+
|
| 56 |
+
# Benchmark Results
|
| 57 |
+
BenchmarkDotNet.Artifacts/
|
| 58 |
+
|
| 59 |
+
# .NET Core
|
| 60 |
+
project.lock.json
|
| 61 |
+
project.fragment.lock.json
|
| 62 |
+
artifacts/
|
| 63 |
+
|
| 64 |
+
# StyleCop
|
| 65 |
+
StyleCopReport.xml
|
| 66 |
+
|
| 67 |
+
# Files built by Visual Studio
|
| 68 |
+
*_i.c
|
| 69 |
+
*_p.c
|
| 70 |
+
*_h.h
|
| 71 |
+
*.ilk
|
| 72 |
+
*.meta
|
| 73 |
+
*.obj
|
| 74 |
+
*.iobj
|
| 75 |
+
*.pch
|
| 76 |
+
*.pdb
|
| 77 |
+
*.ipdb
|
| 78 |
+
*.pgc
|
| 79 |
+
*.pgd
|
| 80 |
+
*.rsp
|
| 81 |
+
*.sbr
|
| 82 |
+
*.tlb
|
| 83 |
+
*.tli
|
| 84 |
+
*.tlh
|
| 85 |
+
*.tmp
|
| 86 |
+
*.tmp_proj
|
| 87 |
+
*_wpftmp.csproj
|
| 88 |
+
*.log
|
| 89 |
+
*.vspscc
|
| 90 |
+
*.vssscc
|
| 91 |
+
.builds
|
| 92 |
+
*.pidb
|
| 93 |
+
*.svclog
|
| 94 |
+
*.scc
|
| 95 |
+
|
| 96 |
+
# Chutzpah Test files
|
| 97 |
+
_Chutzpah*
|
| 98 |
+
|
| 99 |
+
# Visual C++ cache files
|
| 100 |
+
ipch/
|
| 101 |
+
*.aps
|
| 102 |
+
*.ncb
|
| 103 |
+
*.opendb
|
| 104 |
+
*.opensdf
|
| 105 |
+
*.sdf
|
| 106 |
+
*.cachefile
|
| 107 |
+
*.VC.db
|
| 108 |
+
*.VC.VC.opendb
|
| 109 |
+
|
| 110 |
+
# Visual Studio profiler
|
| 111 |
+
*.psess
|
| 112 |
+
*.vsp
|
| 113 |
+
*.vspx
|
| 114 |
+
*.sap
|
| 115 |
+
|
| 116 |
+
# Visual Studio Trace Files
|
| 117 |
+
*.e2e
|
| 118 |
+
|
| 119 |
+
# TFS 2012 Local Workspace
|
| 120 |
+
$tf/
|
| 121 |
+
|
| 122 |
+
# Guidance Automation Toolkit
|
| 123 |
+
*.gpState
|
| 124 |
+
|
| 125 |
+
# ReSharper is a .NET coding add-in
|
| 126 |
+
_ReSharper*/
|
| 127 |
+
*.[Rr]e[Ss]harper
|
| 128 |
+
*.DotSettings.user
|
| 129 |
+
|
| 130 |
+
# TeamCity is a build add-in
|
| 131 |
+
_TeamCity*
|
| 132 |
+
|
| 133 |
+
# DotCover is a Code Coverage Tool
|
| 134 |
+
*.dotCover
|
| 135 |
+
|
| 136 |
+
# AxoCover is a Code Coverage Tool
|
| 137 |
+
.axoCover/*
|
| 138 |
+
!.axoCover/settings.json
|
| 139 |
+
|
| 140 |
+
# Visual Studio code coverage results
|
| 141 |
+
*.coverage
|
| 142 |
+
*.coveragexml
|
| 143 |
+
|
| 144 |
+
# NCrunch
|
| 145 |
+
_NCrunch_*
|
| 146 |
+
.*crunch*.local.xml
|
| 147 |
+
nCrunchTemp_*
|
| 148 |
+
|
| 149 |
+
# MightyMoose
|
| 150 |
+
*.mm.*
|
| 151 |
+
AutoTest.Net/
|
| 152 |
+
|
| 153 |
+
# Web workbench (sass)
|
| 154 |
+
.sass-cache/
|
| 155 |
+
|
| 156 |
+
# Installshield output folder
|
| 157 |
+
[Ee]xpress/
|
| 158 |
+
|
| 159 |
+
# DocProject is a documentation generator add-in
|
| 160 |
+
DocProject/buildhelp/
|
| 161 |
+
DocProject/Help/*.HxT
|
| 162 |
+
DocProject/Help/*.HxC
|
| 163 |
+
DocProject/Help/*.hhc
|
| 164 |
+
DocProject/Help/*.hhk
|
| 165 |
+
DocProject/Help/*.hhp
|
| 166 |
+
DocProject/Help/Html2
|
| 167 |
+
DocProject/Help/html
|
| 168 |
+
|
| 169 |
+
# Click-Once directory
|
| 170 |
+
publish/
|
| 171 |
+
|
| 172 |
+
# Publish Web Output
|
| 173 |
+
*.[Pp]ublish.xml
|
| 174 |
+
*.azurePubxml
|
| 175 |
+
# Note: Comment the next line if you want to checkin your web deploy settings,
|
| 176 |
+
# but database connection strings (with potential passwords) will be unencrypted
|
| 177 |
+
*.pubxml
|
| 178 |
+
*.publishproj
|
| 179 |
+
|
| 180 |
+
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
| 181 |
+
# checkin your Azure Web App publish settings, but sensitive information contained
|
| 182 |
+
# in these scripts will be unencrypted
|
| 183 |
+
PublishScripts/
|
| 184 |
+
|
| 185 |
+
# NuGet Packages
|
| 186 |
+
*.nupkg
|
| 187 |
+
# NuGet Symbol Packages
|
| 188 |
+
*.snupkg
|
| 189 |
+
# The packages folder can be ignored because of Package Restore
|
| 190 |
+
**/[Pp]ackages/*
|
| 191 |
+
# except build/, which is used as an MSBuild target.
|
| 192 |
+
!**/[Pp]ackages/build/
|
| 193 |
+
# Uncomment if necessary however generally it will be regenerated when needed
|
| 194 |
+
#!**/[Pp]ackages/repositories.config
|
| 195 |
+
# NuGet v3's project.json files produces more ignorable files
|
| 196 |
+
*.nuget.props
|
| 197 |
+
*.nuget.targets
|
| 198 |
+
|
| 199 |
+
# Microsoft Azure Build Output
|
| 200 |
+
csx/
|
| 201 |
+
*.build.csdef
|
| 202 |
+
|
| 203 |
+
# Microsoft Azure Emulator
|
| 204 |
+
ecf/
|
| 205 |
+
rcf/
|
| 206 |
+
|
| 207 |
+
# Windows Store app package directories and files
|
| 208 |
+
AppPackages/
|
| 209 |
+
BundleArtifacts/
|
| 210 |
+
Package.StoreAssociation.xml
|
| 211 |
+
_pkginfo.txt
|
| 212 |
+
*.appx
|
| 213 |
+
*.appxbundle
|
| 214 |
+
*.appxupload
|
| 215 |
+
|
| 216 |
+
# Visual Studio cache files
|
| 217 |
+
# files ending in .cache can be ignored
|
| 218 |
+
*.[Cc]ache
|
| 219 |
+
# but keep track of directories ending in .cache
|
| 220 |
+
!?*.[Cc]ache/
|
| 221 |
+
|
| 222 |
+
# Others
|
| 223 |
+
ClientBin/
|
| 224 |
+
~$*
|
| 225 |
+
*~
|
| 226 |
+
*.dbmdl
|
| 227 |
+
*.dbproj.schemaview
|
| 228 |
+
*.jfm
|
| 229 |
+
*.pfx
|
| 230 |
+
*.publishsettings
|
| 231 |
+
orleans.codegen.cs
|
| 232 |
+
|
| 233 |
+
# Including strong name files can present a security risk
|
| 234 |
+
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
|
| 235 |
+
#*.snk
|
| 236 |
+
|
| 237 |
+
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
| 238 |
+
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
| 239 |
+
#bower_components/
|
| 240 |
+
|
| 241 |
+
# RIA/Silverlight projects
|
| 242 |
+
Generated_Code/
|
| 243 |
+
|
| 244 |
+
# Backup & report files from converting an old project file
|
| 245 |
+
# to a newer Visual Studio version. Backup files are not needed,
|
| 246 |
+
# because we have git ;-)
|
| 247 |
+
_UpgradeReport_Files/
|
| 248 |
+
Backup*/
|
| 249 |
+
UpgradeLog*.XML
|
| 250 |
+
UpgradeLog*.htm
|
| 251 |
+
ServiceFabricBackup/
|
| 252 |
+
*.rptproj.bak
|
| 253 |
+
|
| 254 |
+
# SQL Server files
|
| 255 |
+
*.mdf
|
| 256 |
+
*.ldf
|
| 257 |
+
*.ndf
|
| 258 |
+
|
| 259 |
+
# Business Intelligence projects
|
| 260 |
+
*.rdl.data
|
| 261 |
+
*.bim.layout
|
| 262 |
+
*.bim_*.settings
|
| 263 |
+
*.rptproj.rsuser
|
| 264 |
+
*- [Bb]ackup.rdl
|
| 265 |
+
*- [Bb]ackup ([0-9]).rdl
|
| 266 |
+
*- [Bb]ackup ([0-9][0-9]).rdl
|
| 267 |
+
|
| 268 |
+
# Microsoft Fakes
|
| 269 |
+
FakesAssemblies/
|
| 270 |
+
|
| 271 |
+
# GhostDoc plugin setting file
|
| 272 |
+
*.GhostDoc.xml
|
| 273 |
+
|
| 274 |
+
# Node.js Tools for Visual Studio
|
| 275 |
+
.ntvs_analysis.dat
|
| 276 |
+
node_modules/
|
| 277 |
+
|
| 278 |
+
# Visual Studio 6 build log
|
| 279 |
+
*.plg
|
| 280 |
+
|
| 281 |
+
# Visual Studio 6 workspace options file
|
| 282 |
+
*.opt
|
| 283 |
+
|
| 284 |
+
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
|
| 285 |
+
*.vbw
|
| 286 |
+
|
| 287 |
+
# Visual Studio LightSwitch build output
|
| 288 |
+
**/*.HTMLClient/GeneratedArtifacts
|
| 289 |
+
**/*.DesktopClient/GeneratedArtifacts
|
| 290 |
+
**/*.DesktopClient/ModelManifest.xml
|
| 291 |
+
**/*.Server/GeneratedArtifacts
|
| 292 |
+
**/*.Server/ModelManifest.xml
|
| 293 |
+
_Pvt_Extensions
|
| 294 |
+
|
| 295 |
+
# Paket dependency manager
|
| 296 |
+
.paket/paket.exe
|
| 297 |
+
paket-files/
|
| 298 |
+
|
| 299 |
+
# FAKE - F# Make
|
| 300 |
+
.fake/
|
| 301 |
+
|
| 302 |
+
# CodeRush personal settings
|
| 303 |
+
.cr/personal
|
| 304 |
+
|
| 305 |
+
# Python Tools for Visual Studio (PTVS)
|
| 306 |
+
__pycache__/
|
| 307 |
+
*.pyc
|
| 308 |
+
|
| 309 |
+
# Cake - Uncomment if you are using it
|
| 310 |
+
# tools/**
|
| 311 |
+
# !tools/packages.config
|
| 312 |
+
|
| 313 |
+
# Tabs Studio
|
| 314 |
+
*.tss
|
| 315 |
+
|
| 316 |
+
# Telerik's JustMock configuration file
|
| 317 |
+
*.jmconfig
|
| 318 |
+
|
| 319 |
+
# BizTalk build output
|
| 320 |
+
*.btp.cs
|
| 321 |
+
*.btm.cs
|
| 322 |
+
*.odx.cs
|
| 323 |
+
*.xsd.cs
|
| 324 |
+
|
| 325 |
+
# OpenCover UI analysis results
|
| 326 |
+
OpenCover/
|
| 327 |
+
|
| 328 |
+
# Azure Stream Analytics local run output
|
| 329 |
+
ASALocalRun/
|
| 330 |
+
|
| 331 |
+
# MSBuild Binary and Structured Log
|
| 332 |
+
*.binlog
|
| 333 |
+
|
| 334 |
+
# NVidia Nsight GPU debugger configuration file
|
| 335 |
+
*.nvuser
|
| 336 |
+
|
| 337 |
+
# MFractors (Xamarin productivity tool) working folder
|
| 338 |
+
.mfractor/
|
| 339 |
+
|
| 340 |
+
# Local History for Visual Studio
|
| 341 |
+
.localhistory/
|
| 342 |
+
|
| 343 |
+
# BeatPulse healthcheck temp database
|
| 344 |
+
healthchecksdb
|
| 345 |
+
|
| 346 |
+
# Backup folder for Package Reference Convert tool in Visual Studio 2017
|
| 347 |
+
MigrationBackup/
|
| 348 |
+
|
| 349 |
+
# Ionide (cross platform F# VS Code tools) working folder
|
| 350 |
+
.ionide/
|
| 351 |
+
|
| 352 |
+
vidqa_data/
|
| 353 |
+
media/
|
| 354 |
+
temp/
|
| 355 |
+
legen_burned_*/
|
| 356 |
+
legen_srt_*/
|
| 357 |
+
log-vidqa.txt
|
| 358 |
+
legen_errors.txt
|
| 359 |
+
legen-errors.txt
|
| 360 |
+
.vscode/launch.json
|
| 361 |
+
|
| 362 |
+
**.srt
|
| 363 |
+
**.mp4
|
| 364 |
+
**.webm
|
| 365 |
+
**.mkv
|
| 366 |
+
**.avi
|
| 367 |
+
**.mov
|
| 368 |
+
**.wmv
|
| 369 |
+
**.flv
|
| 370 |
+
**.vob
|
| 371 |
+
**.mts
|
| 372 |
+
**.m2ts
|
| 373 |
+
**.ts
|
| 374 |
+
**.yuv
|
| 375 |
+
**.mpg
|
| 376 |
+
**.mpga
|
| 377 |
+
**.mp2
|
| 378 |
+
**.mpeg
|
| 379 |
+
**.mpe
|
| 380 |
+
**.mpv
|
| 381 |
+
**.m2v
|
| 382 |
+
**.m4v
|
| 383 |
+
**.3gp
|
| 384 |
+
**.3g2
|
| 385 |
+
**.nsv
|
| 386 |
+
**.aa
|
| 387 |
+
**.aac
|
| 388 |
+
**.aax
|
| 389 |
+
**.act
|
| 390 |
+
**.aiff
|
| 391 |
+
**.alac
|
| 392 |
+
**.amr
|
| 393 |
+
**.ape
|
| 394 |
+
**.au
|
| 395 |
+
**.awb
|
| 396 |
+
**.dss
|
| 397 |
+
**.dvf
|
| 398 |
+
**.flac
|
| 399 |
+
**.gsm
|
| 400 |
+
**.iklax
|
| 401 |
+
**.ivs
|
| 402 |
+
**.m4a
|
| 403 |
+
**.m4b
|
| 404 |
+
**.m4p
|
| 405 |
+
**.mmf
|
| 406 |
+
**.mp3
|
| 407 |
+
**.mpc
|
| 408 |
+
**.msv
|
| 409 |
+
**.nmf
|
| 410 |
+
**.ogg
|
| 411 |
+
**.oga
|
| 412 |
+
**.mogg
|
| 413 |
+
**.opus
|
| 414 |
+
**.ra
|
| 415 |
+
**.rm
|
| 416 |
+
**.raw
|
| 417 |
+
**.rf64
|
| 418 |
+
**.sln
|
| 419 |
+
**.tta
|
| 420 |
+
**.voc
|
| 421 |
+
**.vox
|
| 422 |
+
**.wav
|
| 423 |
+
**.wma
|
| 424 |
+
**.wv
|
| 425 |
+
**.webm
|
| 426 |
+
**.8svx
|
| 427 |
+
**.mts
|
| 428 |
+
|
| 429 |
+
.venv/
|
| 430 |
+
tmp/
|
| 431 |
+
downloaded/
|
| 432 |
+
downloads/
|
| 433 |
+
build/
|
| 434 |
+
dist/
|
| 435 |
+
*.egg-info/
|
.python-version
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
3.12.12
|
Dockerfile
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.12-slim
|
| 2 |
+
|
| 3 |
+
ARG PYTORCH_INSTALL_CUDA=true
|
| 4 |
+
ARG PYTORCH_CUDA_INDEX_URL=https://download.pytorch.org/whl/cu121
|
| 5 |
+
|
| 6 |
+
ENV PYTHONDONTWRITEBYTECODE=1 \
|
| 7 |
+
PYTHONUNBUFFERED=1
|
| 8 |
+
|
| 9 |
+
WORKDIR /app
|
| 10 |
+
|
| 11 |
+
# Install runtime dependencies required by the pipeline and build tooling
|
| 12 |
+
RUN apt-get update \
|
| 13 |
+
&& apt-get install -y --no-install-recommends ffmpeg git \
|
| 14 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 15 |
+
|
| 16 |
+
COPY requirements.txt ./
|
| 17 |
+
RUN pip install --no-cache-dir --upgrade pip \
|
| 18 |
+
&& pip install --no-cache-dir -r requirements.txt \
|
| 19 |
+
&& if [ "${PYTORCH_INSTALL_CUDA}" = "true" ] && [ -n "${PYTORCH_CUDA_INDEX_URL}" ]; then \
|
| 20 |
+
pip install --no-cache-dir --upgrade torch --index-url "${PYTORCH_CUDA_INDEX_URL}"; \
|
| 21 |
+
fi
|
| 22 |
+
|
| 23 |
+
COPY . .
|
| 24 |
+
|
| 25 |
+
ENTRYPOINT ["python", "legen.py"]
|
| 26 |
+
CMD ["--help"]
|
LICENSE
CHANGED
|
@@ -1,674 +1,674 @@
|
|
| 1 |
-
GNU GENERAL PUBLIC LICENSE
|
| 2 |
-
Version 3, 29 June 2007
|
| 3 |
-
|
| 4 |
-
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
| 5 |
-
Everyone is permitted to copy and distribute verbatim copies
|
| 6 |
-
of this license document, but changing it is not allowed.
|
| 7 |
-
|
| 8 |
-
Preamble
|
| 9 |
-
|
| 10 |
-
The GNU General Public License is a free, copyleft license for
|
| 11 |
-
software and other kinds of works.
|
| 12 |
-
|
| 13 |
-
The licenses for most software and other practical works are designed
|
| 14 |
-
to take away your freedom to share and change the works. By contrast,
|
| 15 |
-
the GNU General Public License is intended to guarantee your freedom to
|
| 16 |
-
share and change all versions of a program--to make sure it remains free
|
| 17 |
-
software for all its users. We, the Free Software Foundation, use the
|
| 18 |
-
GNU General Public License for most of our software; it applies also to
|
| 19 |
-
any other work released this way by its authors. You can apply it to
|
| 20 |
-
your programs, too.
|
| 21 |
-
|
| 22 |
-
When we speak of free software, we are referring to freedom, not
|
| 23 |
-
price. Our General Public Licenses are designed to make sure that you
|
| 24 |
-
have the freedom to distribute copies of free software (and charge for
|
| 25 |
-
them if you wish), that you receive source code or can get it if you
|
| 26 |
-
want it, that you can change the software or use pieces of it in new
|
| 27 |
-
free programs, and that you know you can do these things.
|
| 28 |
-
|
| 29 |
-
To protect your rights, we need to prevent others from denying you
|
| 30 |
-
these rights or asking you to surrender the rights. Therefore, you have
|
| 31 |
-
certain responsibilities if you distribute copies of the software, or if
|
| 32 |
-
you modify it: responsibilities to respect the freedom of others.
|
| 33 |
-
|
| 34 |
-
For example, if you distribute copies of such a program, whether
|
| 35 |
-
gratis or for a fee, you must pass on to the recipients the same
|
| 36 |
-
freedoms that you received. You must make sure that they, too, receive
|
| 37 |
-
or can get the source code. And you must show them these terms so they
|
| 38 |
-
know their rights.
|
| 39 |
-
|
| 40 |
-
Developers that use the GNU GPL protect your rights with two steps:
|
| 41 |
-
(1) assert copyright on the software, and (2) offer you this License
|
| 42 |
-
giving you legal permission to copy, distribute and/or modify it.
|
| 43 |
-
|
| 44 |
-
For the developers' and authors' protection, the GPL clearly explains
|
| 45 |
-
that there is no warranty for this free software. For both users' and
|
| 46 |
-
authors' sake, the GPL requires that modified versions be marked as
|
| 47 |
-
changed, so that their problems will not be attributed erroneously to
|
| 48 |
-
authors of previous versions.
|
| 49 |
-
|
| 50 |
-
Some devices are designed to deny users access to install or run
|
| 51 |
-
modified versions of the software inside them, although the manufacturer
|
| 52 |
-
can do so. This is fundamentally incompatible with the aim of
|
| 53 |
-
protecting users' freedom to change the software. The systematic
|
| 54 |
-
pattern of such abuse occurs in the area of products for individuals to
|
| 55 |
-
use, which is precisely where it is most unacceptable. Therefore, we
|
| 56 |
-
have designed this version of the GPL to prohibit the practice for those
|
| 57 |
-
products. If such problems arise substantially in other domains, we
|
| 58 |
-
stand ready to extend this provision to those domains in future versions
|
| 59 |
-
of the GPL, as needed to protect the freedom of users.
|
| 60 |
-
|
| 61 |
-
Finally, every program is threatened constantly by software patents.
|
| 62 |
-
States should not allow patents to restrict development and use of
|
| 63 |
-
software on general-purpose computers, but in those that do, we wish to
|
| 64 |
-
avoid the special danger that patents applied to a free program could
|
| 65 |
-
make it effectively proprietary. To prevent this, the GPL assures that
|
| 66 |
-
patents cannot be used to render the program non-free.
|
| 67 |
-
|
| 68 |
-
The precise terms and conditions for copying, distribution and
|
| 69 |
-
modification follow.
|
| 70 |
-
|
| 71 |
-
TERMS AND CONDITIONS
|
| 72 |
-
|
| 73 |
-
0. Definitions.
|
| 74 |
-
|
| 75 |
-
"This License" refers to version 3 of the GNU General Public License.
|
| 76 |
-
|
| 77 |
-
"Copyright" also means copyright-like laws that apply to other kinds of
|
| 78 |
-
works, such as semiconductor masks.
|
| 79 |
-
|
| 80 |
-
"The Program" refers to any copyrightable work licensed under this
|
| 81 |
-
License. Each licensee is addressed as "you". "Licensees" and
|
| 82 |
-
"recipients" may be individuals or organizations.
|
| 83 |
-
|
| 84 |
-
To "modify" a work means to copy from or adapt all or part of the work
|
| 85 |
-
in a fashion requiring copyright permission, other than the making of an
|
| 86 |
-
exact copy. The resulting work is called a "modified version" of the
|
| 87 |
-
earlier work or a work "based on" the earlier work.
|
| 88 |
-
|
| 89 |
-
A "covered work" means either the unmodified Program or a work based
|
| 90 |
-
on the Program.
|
| 91 |
-
|
| 92 |
-
To "propagate" a work means to do anything with it that, without
|
| 93 |
-
permission, would make you directly or secondarily liable for
|
| 94 |
-
infringement under applicable copyright law, except executing it on a
|
| 95 |
-
computer or modifying a private copy. Propagation includes copying,
|
| 96 |
-
distribution (with or without modification), making available to the
|
| 97 |
-
public, and in some countries other activities as well.
|
| 98 |
-
|
| 99 |
-
To "convey" a work means any kind of propagation that enables other
|
| 100 |
-
parties to make or receive copies. Mere interaction with a user through
|
| 101 |
-
a computer network, with no transfer of a copy, is not conveying.
|
| 102 |
-
|
| 103 |
-
An interactive user interface displays "Appropriate Legal Notices"
|
| 104 |
-
to the extent that it includes a convenient and prominently visible
|
| 105 |
-
feature that (1) displays an appropriate copyright notice, and (2)
|
| 106 |
-
tells the user that there is no warranty for the work (except to the
|
| 107 |
-
extent that warranties are provided), that licensees may convey the
|
| 108 |
-
work under this License, and how to view a copy of this License. If
|
| 109 |
-
the interface presents a list of user commands or options, such as a
|
| 110 |
-
menu, a prominent item in the list meets this criterion.
|
| 111 |
-
|
| 112 |
-
1. Source Code.
|
| 113 |
-
|
| 114 |
-
The "source code" for a work means the preferred form of the work
|
| 115 |
-
for making modifications to it. "Object code" means any non-source
|
| 116 |
-
form of a work.
|
| 117 |
-
|
| 118 |
-
A "Standard Interface" means an interface that either is an official
|
| 119 |
-
standard defined by a recognized standards body, or, in the case of
|
| 120 |
-
interfaces specified for a particular programming language, one that
|
| 121 |
-
is widely used among developers working in that language.
|
| 122 |
-
|
| 123 |
-
The "System Libraries" of an executable work include anything, other
|
| 124 |
-
than the work as a whole, that (a) is included in the normal form of
|
| 125 |
-
packaging a Major Component, but which is not part of that Major
|
| 126 |
-
Component, and (b) serves only to enable use of the work with that
|
| 127 |
-
Major Component, or to implement a Standard Interface for which an
|
| 128 |
-
implementation is available to the public in source code form. A
|
| 129 |
-
"Major Component", in this context, means a major essential component
|
| 130 |
-
(kernel, window system, and so on) of the specific operating system
|
| 131 |
-
(if any) on which the executable work runs, or a compiler used to
|
| 132 |
-
produce the work, or an object code interpreter used to run it.
|
| 133 |
-
|
| 134 |
-
The "Corresponding Source" for a work in object code form means all
|
| 135 |
-
the source code needed to generate, install, and (for an executable
|
| 136 |
-
work) run the object code and to modify the work, including scripts to
|
| 137 |
-
control those activities. However, it does not include the work's
|
| 138 |
-
System Libraries, or general-purpose tools or generally available free
|
| 139 |
-
programs which are used unmodified in performing those activities but
|
| 140 |
-
which are not part of the work. For example, Corresponding Source
|
| 141 |
-
includes interface definition files associated with source files for
|
| 142 |
-
the work, and the source code for shared libraries and dynamically
|
| 143 |
-
linked subprograms that the work is specifically designed to require,
|
| 144 |
-
such as by intimate data communication or control flow between those
|
| 145 |
-
subprograms and other parts of the work.
|
| 146 |
-
|
| 147 |
-
The Corresponding Source need not include anything that users
|
| 148 |
-
can regenerate automatically from other parts of the Corresponding
|
| 149 |
-
Source.
|
| 150 |
-
|
| 151 |
-
The Corresponding Source for a work in source code form is that
|
| 152 |
-
same work.
|
| 153 |
-
|
| 154 |
-
2. Basic Permissions.
|
| 155 |
-
|
| 156 |
-
All rights granted under this License are granted for the term of
|
| 157 |
-
copyright on the Program, and are irrevocable provided the stated
|
| 158 |
-
conditions are met. This License explicitly affirms your unlimited
|
| 159 |
-
permission to run the unmodified Program. The output from running a
|
| 160 |
-
covered work is covered by this License only if the output, given its
|
| 161 |
-
content, constitutes a covered work. This License acknowledges your
|
| 162 |
-
rights of fair use or other equivalent, as provided by copyright law.
|
| 163 |
-
|
| 164 |
-
You may make, run and propagate covered works that you do not
|
| 165 |
-
convey, without conditions so long as your license otherwise remains
|
| 166 |
-
in force. You may convey covered works to others for the sole purpose
|
| 167 |
-
of having them make modifications exclusively for you, or provide you
|
| 168 |
-
with facilities for running those works, provided that you comply with
|
| 169 |
-
the terms of this License in conveying all material for which you do
|
| 170 |
-
not control copyright. Those thus making or running the covered works
|
| 171 |
-
for you must do so exclusively on your behalf, under your direction
|
| 172 |
-
and control, on terms that prohibit them from making any copies of
|
| 173 |
-
your copyrighted material outside their relationship with you.
|
| 174 |
-
|
| 175 |
-
Conveying under any other circumstances is permitted solely under
|
| 176 |
-
the conditions stated below. Sublicensing is not allowed; section 10
|
| 177 |
-
makes it unnecessary.
|
| 178 |
-
|
| 179 |
-
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
| 180 |
-
|
| 181 |
-
No covered work shall be deemed part of an effective technological
|
| 182 |
-
measure under any applicable law fulfilling obligations under article
|
| 183 |
-
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
| 184 |
-
similar laws prohibiting or restricting circumvention of such
|
| 185 |
-
measures.
|
| 186 |
-
|
| 187 |
-
When you convey a covered work, you waive any legal power to forbid
|
| 188 |
-
circumvention of technological measures to the extent such circumvention
|
| 189 |
-
is effected by exercising rights under this License with respect to
|
| 190 |
-
the covered work, and you disclaim any intention to limit operation or
|
| 191 |
-
modification of the work as a means of enforcing, against the work's
|
| 192 |
-
users, your or third parties' legal rights to forbid circumvention of
|
| 193 |
-
technological measures.
|
| 194 |
-
|
| 195 |
-
4. Conveying Verbatim Copies.
|
| 196 |
-
|
| 197 |
-
You may convey verbatim copies of the Program's source code as you
|
| 198 |
-
receive it, in any medium, provided that you conspicuously and
|
| 199 |
-
appropriately publish on each copy an appropriate copyright notice;
|
| 200 |
-
keep intact all notices stating that this License and any
|
| 201 |
-
non-permissive terms added in accord with section 7 apply to the code;
|
| 202 |
-
keep intact all notices of the absence of any warranty; and give all
|
| 203 |
-
recipients a copy of this License along with the Program.
|
| 204 |
-
|
| 205 |
-
You may charge any price or no price for each copy that you convey,
|
| 206 |
-
and you may offer support or warranty protection for a fee.
|
| 207 |
-
|
| 208 |
-
5. Conveying Modified Source Versions.
|
| 209 |
-
|
| 210 |
-
You may convey a work based on the Program, or the modifications to
|
| 211 |
-
produce it from the Program, in the form of source code under the
|
| 212 |
-
terms of section 4, provided that you also meet all of these conditions:
|
| 213 |
-
|
| 214 |
-
a) The work must carry prominent notices stating that you modified
|
| 215 |
-
it, and giving a relevant date.
|
| 216 |
-
|
| 217 |
-
b) The work must carry prominent notices stating that it is
|
| 218 |
-
released under this License and any conditions added under section
|
| 219 |
-
7. This requirement modifies the requirement in section 4 to
|
| 220 |
-
"keep intact all notices".
|
| 221 |
-
|
| 222 |
-
c) You must license the entire work, as a whole, under this
|
| 223 |
-
License to anyone who comes into possession of a copy. This
|
| 224 |
-
License will therefore apply, along with any applicable section 7
|
| 225 |
-
additional terms, to the whole of the work, and all its parts,
|
| 226 |
-
regardless of how they are packaged. This License gives no
|
| 227 |
-
permission to license the work in any other way, but it does not
|
| 228 |
-
invalidate such permission if you have separately received it.
|
| 229 |
-
|
| 230 |
-
d) If the work has interactive user interfaces, each must display
|
| 231 |
-
Appropriate Legal Notices; however, if the Program has interactive
|
| 232 |
-
interfaces that do not display Appropriate Legal Notices, your
|
| 233 |
-
work need not make them do so.
|
| 234 |
-
|
| 235 |
-
A compilation of a covered work with other separate and independent
|
| 236 |
-
works, which are not by their nature extensions of the covered work,
|
| 237 |
-
and which are not combined with it such as to form a larger program,
|
| 238 |
-
in or on a volume of a storage or distribution medium, is called an
|
| 239 |
-
"aggregate" if the compilation and its resulting copyright are not
|
| 240 |
-
used to limit the access or legal rights of the compilation's users
|
| 241 |
-
beyond what the individual works permit. Inclusion of a covered work
|
| 242 |
-
in an aggregate does not cause this License to apply to the other
|
| 243 |
-
parts of the aggregate.
|
| 244 |
-
|
| 245 |
-
6. Conveying Non-Source Forms.
|
| 246 |
-
|
| 247 |
-
You may convey a covered work in object code form under the terms
|
| 248 |
-
of sections 4 and 5, provided that you also convey the
|
| 249 |
-
machine-readable Corresponding Source under the terms of this License,
|
| 250 |
-
in one of these ways:
|
| 251 |
-
|
| 252 |
-
a) Convey the object code in, or embodied in, a physical product
|
| 253 |
-
(including a physical distribution medium), accompanied by the
|
| 254 |
-
Corresponding Source fixed on a durable physical medium
|
| 255 |
-
customarily used for software interchange.
|
| 256 |
-
|
| 257 |
-
b) Convey the object code in, or embodied in, a physical product
|
| 258 |
-
(including a physical distribution medium), accompanied by a
|
| 259 |
-
written offer, valid for at least three years and valid for as
|
| 260 |
-
long as you offer spare parts or customer support for that product
|
| 261 |
-
model, to give anyone who possesses the object code either (1) a
|
| 262 |
-
copy of the Corresponding Source for all the software in the
|
| 263 |
-
product that is covered by this License, on a durable physical
|
| 264 |
-
medium customarily used for software interchange, for a price no
|
| 265 |
-
more than your reasonable cost of physically performing this
|
| 266 |
-
conveying of source, or (2) access to copy the
|
| 267 |
-
Corresponding Source from a network server at no charge.
|
| 268 |
-
|
| 269 |
-
c) Convey individual copies of the object code with a copy of the
|
| 270 |
-
written offer to provide the Corresponding Source. This
|
| 271 |
-
alternative is allowed only occasionally and noncommercially, and
|
| 272 |
-
only if you received the object code with such an offer, in accord
|
| 273 |
-
with subsection 6b.
|
| 274 |
-
|
| 275 |
-
d) Convey the object code by offering access from a designated
|
| 276 |
-
place (gratis or for a charge), and offer equivalent access to the
|
| 277 |
-
Corresponding Source in the same way through the same place at no
|
| 278 |
-
further charge. You need not require recipients to copy the
|
| 279 |
-
Corresponding Source along with the object code. If the place to
|
| 280 |
-
copy the object code is a network server, the Corresponding Source
|
| 281 |
-
may be on a different server (operated by you or a third party)
|
| 282 |
-
that supports equivalent copying facilities, provided you maintain
|
| 283 |
-
clear directions next to the object code saying where to find the
|
| 284 |
-
Corresponding Source. Regardless of what server hosts the
|
| 285 |
-
Corresponding Source, you remain obligated to ensure that it is
|
| 286 |
-
available for as long as needed to satisfy these requirements.
|
| 287 |
-
|
| 288 |
-
e) Convey the object code using peer-to-peer transmission, provided
|
| 289 |
-
you inform other peers where the object code and Corresponding
|
| 290 |
-
Source of the work are being offered to the general public at no
|
| 291 |
-
charge under subsection 6d.
|
| 292 |
-
|
| 293 |
-
A separable portion of the object code, whose source code is excluded
|
| 294 |
-
from the Corresponding Source as a System Library, need not be
|
| 295 |
-
included in conveying the object code work.
|
| 296 |
-
|
| 297 |
-
A "User Product" is either (1) a "consumer product", which means any
|
| 298 |
-
tangible personal property which is normally used for personal, family,
|
| 299 |
-
or household purposes, or (2) anything designed or sold for incorporation
|
| 300 |
-
into a dwelling. In determining whether a product is a consumer product,
|
| 301 |
-
doubtful cases shall be resolved in favor of coverage. For a particular
|
| 302 |
-
product received by a particular user, "normally used" refers to a
|
| 303 |
-
typical or common use of that class of product, regardless of the status
|
| 304 |
-
of the particular user or of the way in which the particular user
|
| 305 |
-
actually uses, or expects or is expected to use, the product. A product
|
| 306 |
-
is a consumer product regardless of whether the product has substantial
|
| 307 |
-
commercial, industrial or non-consumer uses, unless such uses represent
|
| 308 |
-
the only significant mode of use of the product.
|
| 309 |
-
|
| 310 |
-
"Installation Information" for a User Product means any methods,
|
| 311 |
-
procedures, authorization keys, or other information required to install
|
| 312 |
-
and execute modified versions of a covered work in that User Product from
|
| 313 |
-
a modified version of its Corresponding Source. The information must
|
| 314 |
-
suffice to ensure that the continued functioning of the modified object
|
| 315 |
-
code is in no case prevented or interfered with solely because
|
| 316 |
-
modification has been made.
|
| 317 |
-
|
| 318 |
-
If you convey an object code work under this section in, or with, or
|
| 319 |
-
specifically for use in, a User Product, and the conveying occurs as
|
| 320 |
-
part of a transaction in which the right of possession and use of the
|
| 321 |
-
User Product is transferred to the recipient in perpetuity or for a
|
| 322 |
-
fixed term (regardless of how the transaction is characterized), the
|
| 323 |
-
Corresponding Source conveyed under this section must be accompanied
|
| 324 |
-
by the Installation Information. But this requirement does not apply
|
| 325 |
-
if neither you nor any third party retains the ability to install
|
| 326 |
-
modified object code on the User Product (for example, the work has
|
| 327 |
-
been installed in ROM).
|
| 328 |
-
|
| 329 |
-
The requirement to provide Installation Information does not include a
|
| 330 |
-
requirement to continue to provide support service, warranty, or updates
|
| 331 |
-
for a work that has been modified or installed by the recipient, or for
|
| 332 |
-
the User Product in which it has been modified or installed. Access to a
|
| 333 |
-
network may be denied when the modification itself materially and
|
| 334 |
-
adversely affects the operation of the network or violates the rules and
|
| 335 |
-
protocols for communication across the network.
|
| 336 |
-
|
| 337 |
-
Corresponding Source conveyed, and Installation Information provided,
|
| 338 |
-
in accord with this section must be in a format that is publicly
|
| 339 |
-
documented (and with an implementation available to the public in
|
| 340 |
-
source code form), and must require no special password or key for
|
| 341 |
-
unpacking, reading or copying.
|
| 342 |
-
|
| 343 |
-
7. Additional Terms.
|
| 344 |
-
|
| 345 |
-
"Additional permissions" are terms that supplement the terms of this
|
| 346 |
-
License by making exceptions from one or more of its conditions.
|
| 347 |
-
Additional permissions that are applicable to the entire Program shall
|
| 348 |
-
be treated as though they were included in this License, to the extent
|
| 349 |
-
that they are valid under applicable law. If additional permissions
|
| 350 |
-
apply only to part of the Program, that part may be used separately
|
| 351 |
-
under those permissions, but the entire Program remains governed by
|
| 352 |
-
this License without regard to the additional permissions.
|
| 353 |
-
|
| 354 |
-
When you convey a copy of a covered work, you may at your option
|
| 355 |
-
remove any additional permissions from that copy, or from any part of
|
| 356 |
-
it. (Additional permissions may be written to require their own
|
| 357 |
-
removal in certain cases when you modify the work.) You may place
|
| 358 |
-
additional permissions on material, added by you to a covered work,
|
| 359 |
-
for which you have or can give appropriate copyright permission.
|
| 360 |
-
|
| 361 |
-
Notwithstanding any other provision of this License, for material you
|
| 362 |
-
add to a covered work, you may (if authorized by the copyright holders of
|
| 363 |
-
that material) supplement the terms of this License with terms:
|
| 364 |
-
|
| 365 |
-
a) Disclaiming warranty or limiting liability differently from the
|
| 366 |
-
terms of sections 15 and 16 of this License; or
|
| 367 |
-
|
| 368 |
-
b) Requiring preservation of specified reasonable legal notices or
|
| 369 |
-
author attributions in that material or in the Appropriate Legal
|
| 370 |
-
Notices displayed by works containing it; or
|
| 371 |
-
|
| 372 |
-
c) Prohibiting misrepresentation of the origin of that material, or
|
| 373 |
-
requiring that modified versions of such material be marked in
|
| 374 |
-
reasonable ways as different from the original version; or
|
| 375 |
-
|
| 376 |
-
d) Limiting the use for publicity purposes of names of licensors or
|
| 377 |
-
authors of the material; or
|
| 378 |
-
|
| 379 |
-
e) Declining to grant rights under trademark law for use of some
|
| 380 |
-
trade names, trademarks, or service marks; or
|
| 381 |
-
|
| 382 |
-
f) Requiring indemnification of licensors and authors of that
|
| 383 |
-
material by anyone who conveys the material (or modified versions of
|
| 384 |
-
it) with contractual assumptions of liability to the recipient, for
|
| 385 |
-
any liability that these contractual assumptions directly impose on
|
| 386 |
-
those licensors and authors.
|
| 387 |
-
|
| 388 |
-
All other non-permissive additional terms are considered "further
|
| 389 |
-
restrictions" within the meaning of section 10. If the Program as you
|
| 390 |
-
received it, or any part of it, contains a notice stating that it is
|
| 391 |
-
governed by this License along with a term that is a further
|
| 392 |
-
restriction, you may remove that term. If a license document contains
|
| 393 |
-
a further restriction but permits relicensing or conveying under this
|
| 394 |
-
License, you may add to a covered work material governed by the terms
|
| 395 |
-
of that license document, provided that the further restriction does
|
| 396 |
-
not survive such relicensing or conveying.
|
| 397 |
-
|
| 398 |
-
If you add terms to a covered work in accord with this section, you
|
| 399 |
-
must place, in the relevant source files, a statement of the
|
| 400 |
-
additional terms that apply to those files, or a notice indicating
|
| 401 |
-
where to find the applicable terms.
|
| 402 |
-
|
| 403 |
-
Additional terms, permissive or non-permissive, may be stated in the
|
| 404 |
-
form of a separately written license, or stated as exceptions;
|
| 405 |
-
the above requirements apply either way.
|
| 406 |
-
|
| 407 |
-
8. Termination.
|
| 408 |
-
|
| 409 |
-
You may not propagate or modify a covered work except as expressly
|
| 410 |
-
provided under this License. Any attempt otherwise to propagate or
|
| 411 |
-
modify it is void, and will automatically terminate your rights under
|
| 412 |
-
this License (including any patent licenses granted under the third
|
| 413 |
-
paragraph of section 11).
|
| 414 |
-
|
| 415 |
-
However, if you cease all violation of this License, then your
|
| 416 |
-
license from a particular copyright holder is reinstated (a)
|
| 417 |
-
provisionally, unless and until the copyright holder explicitly and
|
| 418 |
-
finally terminates your license, and (b) permanently, if the copyright
|
| 419 |
-
holder fails to notify you of the violation by some reasonable means
|
| 420 |
-
prior to 60 days after the cessation.
|
| 421 |
-
|
| 422 |
-
Moreover, your license from a particular copyright holder is
|
| 423 |
-
reinstated permanently if the copyright holder notifies you of the
|
| 424 |
-
violation by some reasonable means, this is the first time you have
|
| 425 |
-
received notice of violation of this License (for any work) from that
|
| 426 |
-
copyright holder, and you cure the violation prior to 30 days after
|
| 427 |
-
your receipt of the notice.
|
| 428 |
-
|
| 429 |
-
Termination of your rights under this section does not terminate the
|
| 430 |
-
licenses of parties who have received copies or rights from you under
|
| 431 |
-
this License. If your rights have been terminated and not permanently
|
| 432 |
-
reinstated, you do not qualify to receive new licenses for the same
|
| 433 |
-
material under section 10.
|
| 434 |
-
|
| 435 |
-
9. Acceptance Not Required for Having Copies.
|
| 436 |
-
|
| 437 |
-
You are not required to accept this License in order to receive or
|
| 438 |
-
run a copy of the Program. Ancillary propagation of a covered work
|
| 439 |
-
occurring solely as a consequence of using peer-to-peer transmission
|
| 440 |
-
to receive a copy likewise does not require acceptance. However,
|
| 441 |
-
nothing other than this License grants you permission to propagate or
|
| 442 |
-
modify any covered work. These actions infringe copyright if you do
|
| 443 |
-
not accept this License. Therefore, by modifying or propagating a
|
| 444 |
-
covered work, you indicate your acceptance of this License to do so.
|
| 445 |
-
|
| 446 |
-
10. Automatic Licensing of Downstream Recipients.
|
| 447 |
-
|
| 448 |
-
Each time you convey a covered work, the recipient automatically
|
| 449 |
-
receives a license from the original licensors, to run, modify and
|
| 450 |
-
propagate that work, subject to this License. You are not responsible
|
| 451 |
-
for enforcing compliance by third parties with this License.
|
| 452 |
-
|
| 453 |
-
An "entity transaction" is a transaction transferring control of an
|
| 454 |
-
organization, or substantially all assets of one, or subdividing an
|
| 455 |
-
organization, or merging organizations. If propagation of a covered
|
| 456 |
-
work results from an entity transaction, each party to that
|
| 457 |
-
transaction who receives a copy of the work also receives whatever
|
| 458 |
-
licenses to the work the party's predecessor in interest had or could
|
| 459 |
-
give under the previous paragraph, plus a right to possession of the
|
| 460 |
-
Corresponding Source of the work from the predecessor in interest, if
|
| 461 |
-
the predecessor has it or can get it with reasonable efforts.
|
| 462 |
-
|
| 463 |
-
You may not impose any further restrictions on the exercise of the
|
| 464 |
-
rights granted or affirmed under this License. For example, you may
|
| 465 |
-
not impose a license fee, royalty, or other charge for exercise of
|
| 466 |
-
rights granted under this License, and you may not initiate litigation
|
| 467 |
-
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
| 468 |
-
any patent claim is infringed by making, using, selling, offering for
|
| 469 |
-
sale, or importing the Program or any portion of it.
|
| 470 |
-
|
| 471 |
-
11. Patents.
|
| 472 |
-
|
| 473 |
-
A "contributor" is a copyright holder who authorizes use under this
|
| 474 |
-
License of the Program or a work on which the Program is based. The
|
| 475 |
-
work thus licensed is called the contributor's "contributor version".
|
| 476 |
-
|
| 477 |
-
A contributor's "essential patent claims" are all patent claims
|
| 478 |
-
owned or controlled by the contributor, whether already acquired or
|
| 479 |
-
hereafter acquired, that would be infringed by some manner, permitted
|
| 480 |
-
by this License, of making, using, or selling its contributor version,
|
| 481 |
-
but do not include claims that would be infringed only as a
|
| 482 |
-
consequence of further modification of the contributor version. For
|
| 483 |
-
purposes of this definition, "control" includes the right to grant
|
| 484 |
-
patent sublicenses in a manner consistent with the requirements of
|
| 485 |
-
this License.
|
| 486 |
-
|
| 487 |
-
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
| 488 |
-
patent license under the contributor's essential patent claims, to
|
| 489 |
-
make, use, sell, offer for sale, import and otherwise run, modify and
|
| 490 |
-
propagate the contents of its contributor version.
|
| 491 |
-
|
| 492 |
-
In the following three paragraphs, a "patent license" is any express
|
| 493 |
-
agreement or commitment, however denominated, not to enforce a patent
|
| 494 |
-
(such as an express permission to practice a patent or covenant not to
|
| 495 |
-
sue for patent infringement). To "grant" such a patent license to a
|
| 496 |
-
party means to make such an agreement or commitment not to enforce a
|
| 497 |
-
patent against the party.
|
| 498 |
-
|
| 499 |
-
If you convey a covered work, knowingly relying on a patent license,
|
| 500 |
-
and the Corresponding Source of the work is not available for anyone
|
| 501 |
-
to copy, free of charge and under the terms of this License, through a
|
| 502 |
-
publicly available network server or other readily accessible means,
|
| 503 |
-
then you must either (1) cause the Corresponding Source to be so
|
| 504 |
-
available, or (2) arrange to deprive yourself of the benefit of the
|
| 505 |
-
patent license for this particular work, or (3) arrange, in a manner
|
| 506 |
-
consistent with the requirements of this License, to extend the patent
|
| 507 |
-
license to downstream recipients. "Knowingly relying" means you have
|
| 508 |
-
actual knowledge that, but for the patent license, your conveying the
|
| 509 |
-
covered work in a country, or your recipient's use of the covered work
|
| 510 |
-
in a country, would infringe one or more identifiable patents in that
|
| 511 |
-
country that you have reason to believe are valid.
|
| 512 |
-
|
| 513 |
-
If, pursuant to or in connection with a single transaction or
|
| 514 |
-
arrangement, you convey, or propagate by procuring conveyance of, a
|
| 515 |
-
covered work, and grant a patent license to some of the parties
|
| 516 |
-
receiving the covered work authorizing them to use, propagate, modify
|
| 517 |
-
or convey a specific copy of the covered work, then the patent license
|
| 518 |
-
you grant is automatically extended to all recipients of the covered
|
| 519 |
-
work and works based on it.
|
| 520 |
-
|
| 521 |
-
A patent license is "discriminatory" if it does not include within
|
| 522 |
-
the scope of its coverage, prohibits the exercise of, or is
|
| 523 |
-
conditioned on the non-exercise of one or more of the rights that are
|
| 524 |
-
specifically granted under this License. You may not convey a covered
|
| 525 |
-
work if you are a party to an arrangement with a third party that is
|
| 526 |
-
in the business of distributing software, under which you make payment
|
| 527 |
-
to the third party based on the extent of your activity of conveying
|
| 528 |
-
the work, and under which the third party grants, to any of the
|
| 529 |
-
parties who would receive the covered work from you, a discriminatory
|
| 530 |
-
patent license (a) in connection with copies of the covered work
|
| 531 |
-
conveyed by you (or copies made from those copies), or (b) primarily
|
| 532 |
-
for and in connection with specific products or compilations that
|
| 533 |
-
contain the covered work, unless you entered into that arrangement,
|
| 534 |
-
or that patent license was granted, prior to 28 March 2007.
|
| 535 |
-
|
| 536 |
-
Nothing in this License shall be construed as excluding or limiting
|
| 537 |
-
any implied license or other defenses to infringement that may
|
| 538 |
-
otherwise be available to you under applicable patent law.
|
| 539 |
-
|
| 540 |
-
12. No Surrender of Others' Freedom.
|
| 541 |
-
|
| 542 |
-
If conditions are imposed on you (whether by court order, agreement or
|
| 543 |
-
otherwise) that contradict the conditions of this License, they do not
|
| 544 |
-
excuse you from the conditions of this License. If you cannot convey a
|
| 545 |
-
covered work so as to satisfy simultaneously your obligations under this
|
| 546 |
-
License and any other pertinent obligations, then as a consequence you may
|
| 547 |
-
not convey it at all. For example, if you agree to terms that obligate you
|
| 548 |
-
to collect a royalty for further conveying from those to whom you convey
|
| 549 |
-
the Program, the only way you could satisfy both those terms and this
|
| 550 |
-
License would be to refrain entirely from conveying the Program.
|
| 551 |
-
|
| 552 |
-
13. Use with the GNU Affero General Public License.
|
| 553 |
-
|
| 554 |
-
Notwithstanding any other provision of this License, you have
|
| 555 |
-
permission to link or combine any covered work with a work licensed
|
| 556 |
-
under version 3 of the GNU Affero General Public License into a single
|
| 557 |
-
combined work, and to convey the resulting work. The terms of this
|
| 558 |
-
License will continue to apply to the part which is the covered work,
|
| 559 |
-
but the special requirements of the GNU Affero General Public License,
|
| 560 |
-
section 13, concerning interaction through a network will apply to the
|
| 561 |
-
combination as such.
|
| 562 |
-
|
| 563 |
-
14. Revised Versions of this License.
|
| 564 |
-
|
| 565 |
-
The Free Software Foundation may publish revised and/or new versions of
|
| 566 |
-
the GNU General Public License from time to time. Such new versions will
|
| 567 |
-
be similar in spirit to the present version, but may differ in detail to
|
| 568 |
-
address new problems or concerns.
|
| 569 |
-
|
| 570 |
-
Each version is given a distinguishing version number. If the
|
| 571 |
-
Program specifies that a certain numbered version of the GNU General
|
| 572 |
-
Public License "or any later version" applies to it, you have the
|
| 573 |
-
option of following the terms and conditions either of that numbered
|
| 574 |
-
version or of any later version published by the Free Software
|
| 575 |
-
Foundation. If the Program does not specify a version number of the
|
| 576 |
-
GNU General Public License, you may choose any version ever published
|
| 577 |
-
by the Free Software Foundation.
|
| 578 |
-
|
| 579 |
-
If the Program specifies that a proxy can decide which future
|
| 580 |
-
versions of the GNU General Public License can be used, that proxy's
|
| 581 |
-
public statement of acceptance of a version permanently authorizes you
|
| 582 |
-
to choose that version for the Program.
|
| 583 |
-
|
| 584 |
-
Later license versions may give you additional or different
|
| 585 |
-
permissions. However, no additional obligations are imposed on any
|
| 586 |
-
author or copyright holder as a result of your choosing to follow a
|
| 587 |
-
later version.
|
| 588 |
-
|
| 589 |
-
15. Disclaimer of Warranty.
|
| 590 |
-
|
| 591 |
-
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
| 592 |
-
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
| 593 |
-
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
| 594 |
-
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
| 595 |
-
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
| 596 |
-
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
| 597 |
-
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
| 598 |
-
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
| 599 |
-
|
| 600 |
-
16. Limitation of Liability.
|
| 601 |
-
|
| 602 |
-
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
| 603 |
-
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
| 604 |
-
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
| 605 |
-
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
| 606 |
-
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
| 607 |
-
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
| 608 |
-
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
| 609 |
-
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
| 610 |
-
SUCH DAMAGES.
|
| 611 |
-
|
| 612 |
-
17. Interpretation of Sections 15 and 16.
|
| 613 |
-
|
| 614 |
-
If the disclaimer of warranty and limitation of liability provided
|
| 615 |
-
above cannot be given local legal effect according to their terms,
|
| 616 |
-
reviewing courts shall apply local law that most closely approximates
|
| 617 |
-
an absolute waiver of all civil liability in connection with the
|
| 618 |
-
Program, unless a warranty or assumption of liability accompanies a
|
| 619 |
-
copy of the Program in return for a fee.
|
| 620 |
-
|
| 621 |
-
END OF TERMS AND CONDITIONS
|
| 622 |
-
|
| 623 |
-
How to Apply These Terms to Your New Programs
|
| 624 |
-
|
| 625 |
-
If you develop a new program, and you want it to be of the greatest
|
| 626 |
-
possible use to the public, the best way to achieve this is to make it
|
| 627 |
-
free software which everyone can redistribute and change under these terms.
|
| 628 |
-
|
| 629 |
-
To do so, attach the following notices to the program. It is safest
|
| 630 |
-
to attach them to the start of each source file to most effectively
|
| 631 |
-
state the exclusion of warranty; and each file should have at least
|
| 632 |
-
the "copyright" line and a pointer to where the full notice is found.
|
| 633 |
-
|
| 634 |
-
<one line to give the program's name and a brief idea of what it does.>
|
| 635 |
-
Copyright (C) <year> <name of author>
|
| 636 |
-
|
| 637 |
-
This program is free software: you can redistribute it and/or modify
|
| 638 |
-
it under the terms of the GNU General Public License as published by
|
| 639 |
-
the Free Software Foundation, either version 3 of the License, or
|
| 640 |
-
(at your option) any later version.
|
| 641 |
-
|
| 642 |
-
This program is distributed in the hope that it will be useful,
|
| 643 |
-
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
| 644 |
-
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
| 645 |
-
GNU General Public License for more details.
|
| 646 |
-
|
| 647 |
-
You should have received a copy of the GNU General Public License
|
| 648 |
-
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
| 649 |
-
|
| 650 |
-
Also add information on how to contact you by electronic and paper mail.
|
| 651 |
-
|
| 652 |
-
If the program does terminal interaction, make it output a short
|
| 653 |
-
notice like this when it starts in an interactive mode:
|
| 654 |
-
|
| 655 |
-
<program> Copyright (C) <year> <name of author>
|
| 656 |
-
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
| 657 |
-
This is free software, and you are welcome to redistribute it
|
| 658 |
-
under certain conditions; type `show c' for details.
|
| 659 |
-
|
| 660 |
-
The hypothetical commands `show w' and `show c' should show the appropriate
|
| 661 |
-
parts of the General Public License. Of course, your program's commands
|
| 662 |
-
might be different; for a GUI interface, you would use an "about box".
|
| 663 |
-
|
| 664 |
-
You should also get your employer (if you work as a programmer) or school,
|
| 665 |
-
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
| 666 |
-
For more information on this, and how to apply and follow the GNU GPL, see
|
| 667 |
-
<https://www.gnu.org/licenses/>.
|
| 668 |
-
|
| 669 |
-
The GNU General Public License does not permit incorporating your program
|
| 670 |
-
into proprietary programs. If your program is a subroutine library, you
|
| 671 |
-
may consider it more useful to permit linking proprietary applications with
|
| 672 |
-
the library. If this is what you want to do, use the GNU Lesser General
|
| 673 |
-
Public License instead of this License. But first, please read
|
| 674 |
-
<https://www.gnu.org/licenses/why-not-lgpl.html>.
|
|
|
|
| 1 |
+
GNU GENERAL PUBLIC LICENSE
|
| 2 |
+
Version 3, 29 June 2007
|
| 3 |
+
|
| 4 |
+
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
| 5 |
+
Everyone is permitted to copy and distribute verbatim copies
|
| 6 |
+
of this license document, but changing it is not allowed.
|
| 7 |
+
|
| 8 |
+
Preamble
|
| 9 |
+
|
| 10 |
+
The GNU General Public License is a free, copyleft license for
|
| 11 |
+
software and other kinds of works.
|
| 12 |
+
|
| 13 |
+
The licenses for most software and other practical works are designed
|
| 14 |
+
to take away your freedom to share and change the works. By contrast,
|
| 15 |
+
the GNU General Public License is intended to guarantee your freedom to
|
| 16 |
+
share and change all versions of a program--to make sure it remains free
|
| 17 |
+
software for all its users. We, the Free Software Foundation, use the
|
| 18 |
+
GNU General Public License for most of our software; it applies also to
|
| 19 |
+
any other work released this way by its authors. You can apply it to
|
| 20 |
+
your programs, too.
|
| 21 |
+
|
| 22 |
+
When we speak of free software, we are referring to freedom, not
|
| 23 |
+
price. Our General Public Licenses are designed to make sure that you
|
| 24 |
+
have the freedom to distribute copies of free software (and charge for
|
| 25 |
+
them if you wish), that you receive source code or can get it if you
|
| 26 |
+
want it, that you can change the software or use pieces of it in new
|
| 27 |
+
free programs, and that you know you can do these things.
|
| 28 |
+
|
| 29 |
+
To protect your rights, we need to prevent others from denying you
|
| 30 |
+
these rights or asking you to surrender the rights. Therefore, you have
|
| 31 |
+
certain responsibilities if you distribute copies of the software, or if
|
| 32 |
+
you modify it: responsibilities to respect the freedom of others.
|
| 33 |
+
|
| 34 |
+
For example, if you distribute copies of such a program, whether
|
| 35 |
+
gratis or for a fee, you must pass on to the recipients the same
|
| 36 |
+
freedoms that you received. You must make sure that they, too, receive
|
| 37 |
+
or can get the source code. And you must show them these terms so they
|
| 38 |
+
know their rights.
|
| 39 |
+
|
| 40 |
+
Developers that use the GNU GPL protect your rights with two steps:
|
| 41 |
+
(1) assert copyright on the software, and (2) offer you this License
|
| 42 |
+
giving you legal permission to copy, distribute and/or modify it.
|
| 43 |
+
|
| 44 |
+
For the developers' and authors' protection, the GPL clearly explains
|
| 45 |
+
that there is no warranty for this free software. For both users' and
|
| 46 |
+
authors' sake, the GPL requires that modified versions be marked as
|
| 47 |
+
changed, so that their problems will not be attributed erroneously to
|
| 48 |
+
authors of previous versions.
|
| 49 |
+
|
| 50 |
+
Some devices are designed to deny users access to install or run
|
| 51 |
+
modified versions of the software inside them, although the manufacturer
|
| 52 |
+
can do so. This is fundamentally incompatible with the aim of
|
| 53 |
+
protecting users' freedom to change the software. The systematic
|
| 54 |
+
pattern of such abuse occurs in the area of products for individuals to
|
| 55 |
+
use, which is precisely where it is most unacceptable. Therefore, we
|
| 56 |
+
have designed this version of the GPL to prohibit the practice for those
|
| 57 |
+
products. If such problems arise substantially in other domains, we
|
| 58 |
+
stand ready to extend this provision to those domains in future versions
|
| 59 |
+
of the GPL, as needed to protect the freedom of users.
|
| 60 |
+
|
| 61 |
+
Finally, every program is threatened constantly by software patents.
|
| 62 |
+
States should not allow patents to restrict development and use of
|
| 63 |
+
software on general-purpose computers, but in those that do, we wish to
|
| 64 |
+
avoid the special danger that patents applied to a free program could
|
| 65 |
+
make it effectively proprietary. To prevent this, the GPL assures that
|
| 66 |
+
patents cannot be used to render the program non-free.
|
| 67 |
+
|
| 68 |
+
The precise terms and conditions for copying, distribution and
|
| 69 |
+
modification follow.
|
| 70 |
+
|
| 71 |
+
TERMS AND CONDITIONS
|
| 72 |
+
|
| 73 |
+
0. Definitions.
|
| 74 |
+
|
| 75 |
+
"This License" refers to version 3 of the GNU General Public License.
|
| 76 |
+
|
| 77 |
+
"Copyright" also means copyright-like laws that apply to other kinds of
|
| 78 |
+
works, such as semiconductor masks.
|
| 79 |
+
|
| 80 |
+
"The Program" refers to any copyrightable work licensed under this
|
| 81 |
+
License. Each licensee is addressed as "you". "Licensees" and
|
| 82 |
+
"recipients" may be individuals or organizations.
|
| 83 |
+
|
| 84 |
+
To "modify" a work means to copy from or adapt all or part of the work
|
| 85 |
+
in a fashion requiring copyright permission, other than the making of an
|
| 86 |
+
exact copy. The resulting work is called a "modified version" of the
|
| 87 |
+
earlier work or a work "based on" the earlier work.
|
| 88 |
+
|
| 89 |
+
A "covered work" means either the unmodified Program or a work based
|
| 90 |
+
on the Program.
|
| 91 |
+
|
| 92 |
+
To "propagate" a work means to do anything with it that, without
|
| 93 |
+
permission, would make you directly or secondarily liable for
|
| 94 |
+
infringement under applicable copyright law, except executing it on a
|
| 95 |
+
computer or modifying a private copy. Propagation includes copying,
|
| 96 |
+
distribution (with or without modification), making available to the
|
| 97 |
+
public, and in some countries other activities as well.
|
| 98 |
+
|
| 99 |
+
To "convey" a work means any kind of propagation that enables other
|
| 100 |
+
parties to make or receive copies. Mere interaction with a user through
|
| 101 |
+
a computer network, with no transfer of a copy, is not conveying.
|
| 102 |
+
|
| 103 |
+
An interactive user interface displays "Appropriate Legal Notices"
|
| 104 |
+
to the extent that it includes a convenient and prominently visible
|
| 105 |
+
feature that (1) displays an appropriate copyright notice, and (2)
|
| 106 |
+
tells the user that there is no warranty for the work (except to the
|
| 107 |
+
extent that warranties are provided), that licensees may convey the
|
| 108 |
+
work under this License, and how to view a copy of this License. If
|
| 109 |
+
the interface presents a list of user commands or options, such as a
|
| 110 |
+
menu, a prominent item in the list meets this criterion.
|
| 111 |
+
|
| 112 |
+
1. Source Code.
|
| 113 |
+
|
| 114 |
+
The "source code" for a work means the preferred form of the work
|
| 115 |
+
for making modifications to it. "Object code" means any non-source
|
| 116 |
+
form of a work.
|
| 117 |
+
|
| 118 |
+
A "Standard Interface" means an interface that either is an official
|
| 119 |
+
standard defined by a recognized standards body, or, in the case of
|
| 120 |
+
interfaces specified for a particular programming language, one that
|
| 121 |
+
is widely used among developers working in that language.
|
| 122 |
+
|
| 123 |
+
The "System Libraries" of an executable work include anything, other
|
| 124 |
+
than the work as a whole, that (a) is included in the normal form of
|
| 125 |
+
packaging a Major Component, but which is not part of that Major
|
| 126 |
+
Component, and (b) serves only to enable use of the work with that
|
| 127 |
+
Major Component, or to implement a Standard Interface for which an
|
| 128 |
+
implementation is available to the public in source code form. A
|
| 129 |
+
"Major Component", in this context, means a major essential component
|
| 130 |
+
(kernel, window system, and so on) of the specific operating system
|
| 131 |
+
(if any) on which the executable work runs, or a compiler used to
|
| 132 |
+
produce the work, or an object code interpreter used to run it.
|
| 133 |
+
|
| 134 |
+
The "Corresponding Source" for a work in object code form means all
|
| 135 |
+
the source code needed to generate, install, and (for an executable
|
| 136 |
+
work) run the object code and to modify the work, including scripts to
|
| 137 |
+
control those activities. However, it does not include the work's
|
| 138 |
+
System Libraries, or general-purpose tools or generally available free
|
| 139 |
+
programs which are used unmodified in performing those activities but
|
| 140 |
+
which are not part of the work. For example, Corresponding Source
|
| 141 |
+
includes interface definition files associated with source files for
|
| 142 |
+
the work, and the source code for shared libraries and dynamically
|
| 143 |
+
linked subprograms that the work is specifically designed to require,
|
| 144 |
+
such as by intimate data communication or control flow between those
|
| 145 |
+
subprograms and other parts of the work.
|
| 146 |
+
|
| 147 |
+
The Corresponding Source need not include anything that users
|
| 148 |
+
can regenerate automatically from other parts of the Corresponding
|
| 149 |
+
Source.
|
| 150 |
+
|
| 151 |
+
The Corresponding Source for a work in source code form is that
|
| 152 |
+
same work.
|
| 153 |
+
|
| 154 |
+
2. Basic Permissions.
|
| 155 |
+
|
| 156 |
+
All rights granted under this License are granted for the term of
|
| 157 |
+
copyright on the Program, and are irrevocable provided the stated
|
| 158 |
+
conditions are met. This License explicitly affirms your unlimited
|
| 159 |
+
permission to run the unmodified Program. The output from running a
|
| 160 |
+
covered work is covered by this License only if the output, given its
|
| 161 |
+
content, constitutes a covered work. This License acknowledges your
|
| 162 |
+
rights of fair use or other equivalent, as provided by copyright law.
|
| 163 |
+
|
| 164 |
+
You may make, run and propagate covered works that you do not
|
| 165 |
+
convey, without conditions so long as your license otherwise remains
|
| 166 |
+
in force. You may convey covered works to others for the sole purpose
|
| 167 |
+
of having them make modifications exclusively for you, or provide you
|
| 168 |
+
with facilities for running those works, provided that you comply with
|
| 169 |
+
the terms of this License in conveying all material for which you do
|
| 170 |
+
not control copyright. Those thus making or running the covered works
|
| 171 |
+
for you must do so exclusively on your behalf, under your direction
|
| 172 |
+
and control, on terms that prohibit them from making any copies of
|
| 173 |
+
your copyrighted material outside their relationship with you.
|
| 174 |
+
|
| 175 |
+
Conveying under any other circumstances is permitted solely under
|
| 176 |
+
the conditions stated below. Sublicensing is not allowed; section 10
|
| 177 |
+
makes it unnecessary.
|
| 178 |
+
|
| 179 |
+
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
| 180 |
+
|
| 181 |
+
No covered work shall be deemed part of an effective technological
|
| 182 |
+
measure under any applicable law fulfilling obligations under article
|
| 183 |
+
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
| 184 |
+
similar laws prohibiting or restricting circumvention of such
|
| 185 |
+
measures.
|
| 186 |
+
|
| 187 |
+
When you convey a covered work, you waive any legal power to forbid
|
| 188 |
+
circumvention of technological measures to the extent such circumvention
|
| 189 |
+
is effected by exercising rights under this License with respect to
|
| 190 |
+
the covered work, and you disclaim any intention to limit operation or
|
| 191 |
+
modification of the work as a means of enforcing, against the work's
|
| 192 |
+
users, your or third parties' legal rights to forbid circumvention of
|
| 193 |
+
technological measures.
|
| 194 |
+
|
| 195 |
+
4. Conveying Verbatim Copies.
|
| 196 |
+
|
| 197 |
+
You may convey verbatim copies of the Program's source code as you
|
| 198 |
+
receive it, in any medium, provided that you conspicuously and
|
| 199 |
+
appropriately publish on each copy an appropriate copyright notice;
|
| 200 |
+
keep intact all notices stating that this License and any
|
| 201 |
+
non-permissive terms added in accord with section 7 apply to the code;
|
| 202 |
+
keep intact all notices of the absence of any warranty; and give all
|
| 203 |
+
recipients a copy of this License along with the Program.
|
| 204 |
+
|
| 205 |
+
You may charge any price or no price for each copy that you convey,
|
| 206 |
+
and you may offer support or warranty protection for a fee.
|
| 207 |
+
|
| 208 |
+
5. Conveying Modified Source Versions.
|
| 209 |
+
|
| 210 |
+
You may convey a work based on the Program, or the modifications to
|
| 211 |
+
produce it from the Program, in the form of source code under the
|
| 212 |
+
terms of section 4, provided that you also meet all of these conditions:
|
| 213 |
+
|
| 214 |
+
a) The work must carry prominent notices stating that you modified
|
| 215 |
+
it, and giving a relevant date.
|
| 216 |
+
|
| 217 |
+
b) The work must carry prominent notices stating that it is
|
| 218 |
+
released under this License and any conditions added under section
|
| 219 |
+
7. This requirement modifies the requirement in section 4 to
|
| 220 |
+
"keep intact all notices".
|
| 221 |
+
|
| 222 |
+
c) You must license the entire work, as a whole, under this
|
| 223 |
+
License to anyone who comes into possession of a copy. This
|
| 224 |
+
License will therefore apply, along with any applicable section 7
|
| 225 |
+
additional terms, to the whole of the work, and all its parts,
|
| 226 |
+
regardless of how they are packaged. This License gives no
|
| 227 |
+
permission to license the work in any other way, but it does not
|
| 228 |
+
invalidate such permission if you have separately received it.
|
| 229 |
+
|
| 230 |
+
d) If the work has interactive user interfaces, each must display
|
| 231 |
+
Appropriate Legal Notices; however, if the Program has interactive
|
| 232 |
+
interfaces that do not display Appropriate Legal Notices, your
|
| 233 |
+
work need not make them do so.
|
| 234 |
+
|
| 235 |
+
A compilation of a covered work with other separate and independent
|
| 236 |
+
works, which are not by their nature extensions of the covered work,
|
| 237 |
+
and which are not combined with it such as to form a larger program,
|
| 238 |
+
in or on a volume of a storage or distribution medium, is called an
|
| 239 |
+
"aggregate" if the compilation and its resulting copyright are not
|
| 240 |
+
used to limit the access or legal rights of the compilation's users
|
| 241 |
+
beyond what the individual works permit. Inclusion of a covered work
|
| 242 |
+
in an aggregate does not cause this License to apply to the other
|
| 243 |
+
parts of the aggregate.
|
| 244 |
+
|
| 245 |
+
6. Conveying Non-Source Forms.
|
| 246 |
+
|
| 247 |
+
You may convey a covered work in object code form under the terms
|
| 248 |
+
of sections 4 and 5, provided that you also convey the
|
| 249 |
+
machine-readable Corresponding Source under the terms of this License,
|
| 250 |
+
in one of these ways:
|
| 251 |
+
|
| 252 |
+
a) Convey the object code in, or embodied in, a physical product
|
| 253 |
+
(including a physical distribution medium), accompanied by the
|
| 254 |
+
Corresponding Source fixed on a durable physical medium
|
| 255 |
+
customarily used for software interchange.
|
| 256 |
+
|
| 257 |
+
b) Convey the object code in, or embodied in, a physical product
|
| 258 |
+
(including a physical distribution medium), accompanied by a
|
| 259 |
+
written offer, valid for at least three years and valid for as
|
| 260 |
+
long as you offer spare parts or customer support for that product
|
| 261 |
+
model, to give anyone who possesses the object code either (1) a
|
| 262 |
+
copy of the Corresponding Source for all the software in the
|
| 263 |
+
product that is covered by this License, on a durable physical
|
| 264 |
+
medium customarily used for software interchange, for a price no
|
| 265 |
+
more than your reasonable cost of physically performing this
|
| 266 |
+
conveying of source, or (2) access to copy the
|
| 267 |
+
Corresponding Source from a network server at no charge.
|
| 268 |
+
|
| 269 |
+
c) Convey individual copies of the object code with a copy of the
|
| 270 |
+
written offer to provide the Corresponding Source. This
|
| 271 |
+
alternative is allowed only occasionally and noncommercially, and
|
| 272 |
+
only if you received the object code with such an offer, in accord
|
| 273 |
+
with subsection 6b.
|
| 274 |
+
|
| 275 |
+
d) Convey the object code by offering access from a designated
|
| 276 |
+
place (gratis or for a charge), and offer equivalent access to the
|
| 277 |
+
Corresponding Source in the same way through the same place at no
|
| 278 |
+
further charge. You need not require recipients to copy the
|
| 279 |
+
Corresponding Source along with the object code. If the place to
|
| 280 |
+
copy the object code is a network server, the Corresponding Source
|
| 281 |
+
may be on a different server (operated by you or a third party)
|
| 282 |
+
that supports equivalent copying facilities, provided you maintain
|
| 283 |
+
clear directions next to the object code saying where to find the
|
| 284 |
+
Corresponding Source. Regardless of what server hosts the
|
| 285 |
+
Corresponding Source, you remain obligated to ensure that it is
|
| 286 |
+
available for as long as needed to satisfy these requirements.
|
| 287 |
+
|
| 288 |
+
e) Convey the object code using peer-to-peer transmission, provided
|
| 289 |
+
you inform other peers where the object code and Corresponding
|
| 290 |
+
Source of the work are being offered to the general public at no
|
| 291 |
+
charge under subsection 6d.
|
| 292 |
+
|
| 293 |
+
A separable portion of the object code, whose source code is excluded
|
| 294 |
+
from the Corresponding Source as a System Library, need not be
|
| 295 |
+
included in conveying the object code work.
|
| 296 |
+
|
| 297 |
+
A "User Product" is either (1) a "consumer product", which means any
|
| 298 |
+
tangible personal property which is normally used for personal, family,
|
| 299 |
+
or household purposes, or (2) anything designed or sold for incorporation
|
| 300 |
+
into a dwelling. In determining whether a product is a consumer product,
|
| 301 |
+
doubtful cases shall be resolved in favor of coverage. For a particular
|
| 302 |
+
product received by a particular user, "normally used" refers to a
|
| 303 |
+
typical or common use of that class of product, regardless of the status
|
| 304 |
+
of the particular user or of the way in which the particular user
|
| 305 |
+
actually uses, or expects or is expected to use, the product. A product
|
| 306 |
+
is a consumer product regardless of whether the product has substantial
|
| 307 |
+
commercial, industrial or non-consumer uses, unless such uses represent
|
| 308 |
+
the only significant mode of use of the product.
|
| 309 |
+
|
| 310 |
+
"Installation Information" for a User Product means any methods,
|
| 311 |
+
procedures, authorization keys, or other information required to install
|
| 312 |
+
and execute modified versions of a covered work in that User Product from
|
| 313 |
+
a modified version of its Corresponding Source. The information must
|
| 314 |
+
suffice to ensure that the continued functioning of the modified object
|
| 315 |
+
code is in no case prevented or interfered with solely because
|
| 316 |
+
modification has been made.
|
| 317 |
+
|
| 318 |
+
If you convey an object code work under this section in, or with, or
|
| 319 |
+
specifically for use in, a User Product, and the conveying occurs as
|
| 320 |
+
part of a transaction in which the right of possession and use of the
|
| 321 |
+
User Product is transferred to the recipient in perpetuity or for a
|
| 322 |
+
fixed term (regardless of how the transaction is characterized), the
|
| 323 |
+
Corresponding Source conveyed under this section must be accompanied
|
| 324 |
+
by the Installation Information. But this requirement does not apply
|
| 325 |
+
if neither you nor any third party retains the ability to install
|
| 326 |
+
modified object code on the User Product (for example, the work has
|
| 327 |
+
been installed in ROM).
|
| 328 |
+
|
| 329 |
+
The requirement to provide Installation Information does not include a
|
| 330 |
+
requirement to continue to provide support service, warranty, or updates
|
| 331 |
+
for a work that has been modified or installed by the recipient, or for
|
| 332 |
+
the User Product in which it has been modified or installed. Access to a
|
| 333 |
+
network may be denied when the modification itself materially and
|
| 334 |
+
adversely affects the operation of the network or violates the rules and
|
| 335 |
+
protocols for communication across the network.
|
| 336 |
+
|
| 337 |
+
Corresponding Source conveyed, and Installation Information provided,
|
| 338 |
+
in accord with this section must be in a format that is publicly
|
| 339 |
+
documented (and with an implementation available to the public in
|
| 340 |
+
source code form), and must require no special password or key for
|
| 341 |
+
unpacking, reading or copying.
|
| 342 |
+
|
| 343 |
+
7. Additional Terms.
|
| 344 |
+
|
| 345 |
+
"Additional permissions" are terms that supplement the terms of this
|
| 346 |
+
License by making exceptions from one or more of its conditions.
|
| 347 |
+
Additional permissions that are applicable to the entire Program shall
|
| 348 |
+
be treated as though they were included in this License, to the extent
|
| 349 |
+
that they are valid under applicable law. If additional permissions
|
| 350 |
+
apply only to part of the Program, that part may be used separately
|
| 351 |
+
under those permissions, but the entire Program remains governed by
|
| 352 |
+
this License without regard to the additional permissions.
|
| 353 |
+
|
| 354 |
+
When you convey a copy of a covered work, you may at your option
|
| 355 |
+
remove any additional permissions from that copy, or from any part of
|
| 356 |
+
it. (Additional permissions may be written to require their own
|
| 357 |
+
removal in certain cases when you modify the work.) You may place
|
| 358 |
+
additional permissions on material, added by you to a covered work,
|
| 359 |
+
for which you have or can give appropriate copyright permission.
|
| 360 |
+
|
| 361 |
+
Notwithstanding any other provision of this License, for material you
|
| 362 |
+
add to a covered work, you may (if authorized by the copyright holders of
|
| 363 |
+
that material) supplement the terms of this License with terms:
|
| 364 |
+
|
| 365 |
+
a) Disclaiming warranty or limiting liability differently from the
|
| 366 |
+
terms of sections 15 and 16 of this License; or
|
| 367 |
+
|
| 368 |
+
b) Requiring preservation of specified reasonable legal notices or
|
| 369 |
+
author attributions in that material or in the Appropriate Legal
|
| 370 |
+
Notices displayed by works containing it; or
|
| 371 |
+
|
| 372 |
+
c) Prohibiting misrepresentation of the origin of that material, or
|
| 373 |
+
requiring that modified versions of such material be marked in
|
| 374 |
+
reasonable ways as different from the original version; or
|
| 375 |
+
|
| 376 |
+
d) Limiting the use for publicity purposes of names of licensors or
|
| 377 |
+
authors of the material; or
|
| 378 |
+
|
| 379 |
+
e) Declining to grant rights under trademark law for use of some
|
| 380 |
+
trade names, trademarks, or service marks; or
|
| 381 |
+
|
| 382 |
+
f) Requiring indemnification of licensors and authors of that
|
| 383 |
+
material by anyone who conveys the material (or modified versions of
|
| 384 |
+
it) with contractual assumptions of liability to the recipient, for
|
| 385 |
+
any liability that these contractual assumptions directly impose on
|
| 386 |
+
those licensors and authors.
|
| 387 |
+
|
| 388 |
+
All other non-permissive additional terms are considered "further
|
| 389 |
+
restrictions" within the meaning of section 10. If the Program as you
|
| 390 |
+
received it, or any part of it, contains a notice stating that it is
|
| 391 |
+
governed by this License along with a term that is a further
|
| 392 |
+
restriction, you may remove that term. If a license document contains
|
| 393 |
+
a further restriction but permits relicensing or conveying under this
|
| 394 |
+
License, you may add to a covered work material governed by the terms
|
| 395 |
+
of that license document, provided that the further restriction does
|
| 396 |
+
not survive such relicensing or conveying.
|
| 397 |
+
|
| 398 |
+
If you add terms to a covered work in accord with this section, you
|
| 399 |
+
must place, in the relevant source files, a statement of the
|
| 400 |
+
additional terms that apply to those files, or a notice indicating
|
| 401 |
+
where to find the applicable terms.
|
| 402 |
+
|
| 403 |
+
Additional terms, permissive or non-permissive, may be stated in the
|
| 404 |
+
form of a separately written license, or stated as exceptions;
|
| 405 |
+
the above requirements apply either way.
|
| 406 |
+
|
| 407 |
+
8. Termination.
|
| 408 |
+
|
| 409 |
+
You may not propagate or modify a covered work except as expressly
|
| 410 |
+
provided under this License. Any attempt otherwise to propagate or
|
| 411 |
+
modify it is void, and will automatically terminate your rights under
|
| 412 |
+
this License (including any patent licenses granted under the third
|
| 413 |
+
paragraph of section 11).
|
| 414 |
+
|
| 415 |
+
However, if you cease all violation of this License, then your
|
| 416 |
+
license from a particular copyright holder is reinstated (a)
|
| 417 |
+
provisionally, unless and until the copyright holder explicitly and
|
| 418 |
+
finally terminates your license, and (b) permanently, if the copyright
|
| 419 |
+
holder fails to notify you of the violation by some reasonable means
|
| 420 |
+
prior to 60 days after the cessation.
|
| 421 |
+
|
| 422 |
+
Moreover, your license from a particular copyright holder is
|
| 423 |
+
reinstated permanently if the copyright holder notifies you of the
|
| 424 |
+
violation by some reasonable means, this is the first time you have
|
| 425 |
+
received notice of violation of this License (for any work) from that
|
| 426 |
+
copyright holder, and you cure the violation prior to 30 days after
|
| 427 |
+
your receipt of the notice.
|
| 428 |
+
|
| 429 |
+
Termination of your rights under this section does not terminate the
|
| 430 |
+
licenses of parties who have received copies or rights from you under
|
| 431 |
+
this License. If your rights have been terminated and not permanently
|
| 432 |
+
reinstated, you do not qualify to receive new licenses for the same
|
| 433 |
+
material under section 10.
|
| 434 |
+
|
| 435 |
+
9. Acceptance Not Required for Having Copies.
|
| 436 |
+
|
| 437 |
+
You are not required to accept this License in order to receive or
|
| 438 |
+
run a copy of the Program. Ancillary propagation of a covered work
|
| 439 |
+
occurring solely as a consequence of using peer-to-peer transmission
|
| 440 |
+
to receive a copy likewise does not require acceptance. However,
|
| 441 |
+
nothing other than this License grants you permission to propagate or
|
| 442 |
+
modify any covered work. These actions infringe copyright if you do
|
| 443 |
+
not accept this License. Therefore, by modifying or propagating a
|
| 444 |
+
covered work, you indicate your acceptance of this License to do so.
|
| 445 |
+
|
| 446 |
+
10. Automatic Licensing of Downstream Recipients.
|
| 447 |
+
|
| 448 |
+
Each time you convey a covered work, the recipient automatically
|
| 449 |
+
receives a license from the original licensors, to run, modify and
|
| 450 |
+
propagate that work, subject to this License. You are not responsible
|
| 451 |
+
for enforcing compliance by third parties with this License.
|
| 452 |
+
|
| 453 |
+
An "entity transaction" is a transaction transferring control of an
|
| 454 |
+
organization, or substantially all assets of one, or subdividing an
|
| 455 |
+
organization, or merging organizations. If propagation of a covered
|
| 456 |
+
work results from an entity transaction, each party to that
|
| 457 |
+
transaction who receives a copy of the work also receives whatever
|
| 458 |
+
licenses to the work the party's predecessor in interest had or could
|
| 459 |
+
give under the previous paragraph, plus a right to possession of the
|
| 460 |
+
Corresponding Source of the work from the predecessor in interest, if
|
| 461 |
+
the predecessor has it or can get it with reasonable efforts.
|
| 462 |
+
|
| 463 |
+
You may not impose any further restrictions on the exercise of the
|
| 464 |
+
rights granted or affirmed under this License. For example, you may
|
| 465 |
+
not impose a license fee, royalty, or other charge for exercise of
|
| 466 |
+
rights granted under this License, and you may not initiate litigation
|
| 467 |
+
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
| 468 |
+
any patent claim is infringed by making, using, selling, offering for
|
| 469 |
+
sale, or importing the Program or any portion of it.
|
| 470 |
+
|
| 471 |
+
11. Patents.
|
| 472 |
+
|
| 473 |
+
A "contributor" is a copyright holder who authorizes use under this
|
| 474 |
+
License of the Program or a work on which the Program is based. The
|
| 475 |
+
work thus licensed is called the contributor's "contributor version".
|
| 476 |
+
|
| 477 |
+
A contributor's "essential patent claims" are all patent claims
|
| 478 |
+
owned or controlled by the contributor, whether already acquired or
|
| 479 |
+
hereafter acquired, that would be infringed by some manner, permitted
|
| 480 |
+
by this License, of making, using, or selling its contributor version,
|
| 481 |
+
but do not include claims that would be infringed only as a
|
| 482 |
+
consequence of further modification of the contributor version. For
|
| 483 |
+
purposes of this definition, "control" includes the right to grant
|
| 484 |
+
patent sublicenses in a manner consistent with the requirements of
|
| 485 |
+
this License.
|
| 486 |
+
|
| 487 |
+
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
| 488 |
+
patent license under the contributor's essential patent claims, to
|
| 489 |
+
make, use, sell, offer for sale, import and otherwise run, modify and
|
| 490 |
+
propagate the contents of its contributor version.
|
| 491 |
+
|
| 492 |
+
In the following three paragraphs, a "patent license" is any express
|
| 493 |
+
agreement or commitment, however denominated, not to enforce a patent
|
| 494 |
+
(such as an express permission to practice a patent or covenant not to
|
| 495 |
+
sue for patent infringement). To "grant" such a patent license to a
|
| 496 |
+
party means to make such an agreement or commitment not to enforce a
|
| 497 |
+
patent against the party.
|
| 498 |
+
|
| 499 |
+
If you convey a covered work, knowingly relying on a patent license,
|
| 500 |
+
and the Corresponding Source of the work is not available for anyone
|
| 501 |
+
to copy, free of charge and under the terms of this License, through a
|
| 502 |
+
publicly available network server or other readily accessible means,
|
| 503 |
+
then you must either (1) cause the Corresponding Source to be so
|
| 504 |
+
available, or (2) arrange to deprive yourself of the benefit of the
|
| 505 |
+
patent license for this particular work, or (3) arrange, in a manner
|
| 506 |
+
consistent with the requirements of this License, to extend the patent
|
| 507 |
+
license to downstream recipients. "Knowingly relying" means you have
|
| 508 |
+
actual knowledge that, but for the patent license, your conveying the
|
| 509 |
+
covered work in a country, or your recipient's use of the covered work
|
| 510 |
+
in a country, would infringe one or more identifiable patents in that
|
| 511 |
+
country that you have reason to believe are valid.
|
| 512 |
+
|
| 513 |
+
If, pursuant to or in connection with a single transaction or
|
| 514 |
+
arrangement, you convey, or propagate by procuring conveyance of, a
|
| 515 |
+
covered work, and grant a patent license to some of the parties
|
| 516 |
+
receiving the covered work authorizing them to use, propagate, modify
|
| 517 |
+
or convey a specific copy of the covered work, then the patent license
|
| 518 |
+
you grant is automatically extended to all recipients of the covered
|
| 519 |
+
work and works based on it.
|
| 520 |
+
|
| 521 |
+
A patent license is "discriminatory" if it does not include within
|
| 522 |
+
the scope of its coverage, prohibits the exercise of, or is
|
| 523 |
+
conditioned on the non-exercise of one or more of the rights that are
|
| 524 |
+
specifically granted under this License. You may not convey a covered
|
| 525 |
+
work if you are a party to an arrangement with a third party that is
|
| 526 |
+
in the business of distributing software, under which you make payment
|
| 527 |
+
to the third party based on the extent of your activity of conveying
|
| 528 |
+
the work, and under which the third party grants, to any of the
|
| 529 |
+
parties who would receive the covered work from you, a discriminatory
|
| 530 |
+
patent license (a) in connection with copies of the covered work
|
| 531 |
+
conveyed by you (or copies made from those copies), or (b) primarily
|
| 532 |
+
for and in connection with specific products or compilations that
|
| 533 |
+
contain the covered work, unless you entered into that arrangement,
|
| 534 |
+
or that patent license was granted, prior to 28 March 2007.
|
| 535 |
+
|
| 536 |
+
Nothing in this License shall be construed as excluding or limiting
|
| 537 |
+
any implied license or other defenses to infringement that may
|
| 538 |
+
otherwise be available to you under applicable patent law.
|
| 539 |
+
|
| 540 |
+
12. No Surrender of Others' Freedom.
|
| 541 |
+
|
| 542 |
+
If conditions are imposed on you (whether by court order, agreement or
|
| 543 |
+
otherwise) that contradict the conditions of this License, they do not
|
| 544 |
+
excuse you from the conditions of this License. If you cannot convey a
|
| 545 |
+
covered work so as to satisfy simultaneously your obligations under this
|
| 546 |
+
License and any other pertinent obligations, then as a consequence you may
|
| 547 |
+
not convey it at all. For example, if you agree to terms that obligate you
|
| 548 |
+
to collect a royalty for further conveying from those to whom you convey
|
| 549 |
+
the Program, the only way you could satisfy both those terms and this
|
| 550 |
+
License would be to refrain entirely from conveying the Program.
|
| 551 |
+
|
| 552 |
+
13. Use with the GNU Affero General Public License.
|
| 553 |
+
|
| 554 |
+
Notwithstanding any other provision of this License, you have
|
| 555 |
+
permission to link or combine any covered work with a work licensed
|
| 556 |
+
under version 3 of the GNU Affero General Public License into a single
|
| 557 |
+
combined work, and to convey the resulting work. The terms of this
|
| 558 |
+
License will continue to apply to the part which is the covered work,
|
| 559 |
+
but the special requirements of the GNU Affero General Public License,
|
| 560 |
+
section 13, concerning interaction through a network will apply to the
|
| 561 |
+
combination as such.
|
| 562 |
+
|
| 563 |
+
14. Revised Versions of this License.
|
| 564 |
+
|
| 565 |
+
The Free Software Foundation may publish revised and/or new versions of
|
| 566 |
+
the GNU General Public License from time to time. Such new versions will
|
| 567 |
+
be similar in spirit to the present version, but may differ in detail to
|
| 568 |
+
address new problems or concerns.
|
| 569 |
+
|
| 570 |
+
Each version is given a distinguishing version number. If the
|
| 571 |
+
Program specifies that a certain numbered version of the GNU General
|
| 572 |
+
Public License "or any later version" applies to it, you have the
|
| 573 |
+
option of following the terms and conditions either of that numbered
|
| 574 |
+
version or of any later version published by the Free Software
|
| 575 |
+
Foundation. If the Program does not specify a version number of the
|
| 576 |
+
GNU General Public License, you may choose any version ever published
|
| 577 |
+
by the Free Software Foundation.
|
| 578 |
+
|
| 579 |
+
If the Program specifies that a proxy can decide which future
|
| 580 |
+
versions of the GNU General Public License can be used, that proxy's
|
| 581 |
+
public statement of acceptance of a version permanently authorizes you
|
| 582 |
+
to choose that version for the Program.
|
| 583 |
+
|
| 584 |
+
Later license versions may give you additional or different
|
| 585 |
+
permissions. However, no additional obligations are imposed on any
|
| 586 |
+
author or copyright holder as a result of your choosing to follow a
|
| 587 |
+
later version.
|
| 588 |
+
|
| 589 |
+
15. Disclaimer of Warranty.
|
| 590 |
+
|
| 591 |
+
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
| 592 |
+
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
| 593 |
+
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
| 594 |
+
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
| 595 |
+
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
| 596 |
+
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
| 597 |
+
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
| 598 |
+
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
| 599 |
+
|
| 600 |
+
16. Limitation of Liability.
|
| 601 |
+
|
| 602 |
+
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
| 603 |
+
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
| 604 |
+
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
| 605 |
+
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
| 606 |
+
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
| 607 |
+
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
| 608 |
+
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
| 609 |
+
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
| 610 |
+
SUCH DAMAGES.
|
| 611 |
+
|
| 612 |
+
17. Interpretation of Sections 15 and 16.
|
| 613 |
+
|
| 614 |
+
If the disclaimer of warranty and limitation of liability provided
|
| 615 |
+
above cannot be given local legal effect according to their terms,
|
| 616 |
+
reviewing courts shall apply local law that most closely approximates
|
| 617 |
+
an absolute waiver of all civil liability in connection with the
|
| 618 |
+
Program, unless a warranty or assumption of liability accompanies a
|
| 619 |
+
copy of the Program in return for a fee.
|
| 620 |
+
|
| 621 |
+
END OF TERMS AND CONDITIONS
|
| 622 |
+
|
| 623 |
+
How to Apply These Terms to Your New Programs
|
| 624 |
+
|
| 625 |
+
If you develop a new program, and you want it to be of the greatest
|
| 626 |
+
possible use to the public, the best way to achieve this is to make it
|
| 627 |
+
free software which everyone can redistribute and change under these terms.
|
| 628 |
+
|
| 629 |
+
To do so, attach the following notices to the program. It is safest
|
| 630 |
+
to attach them to the start of each source file to most effectively
|
| 631 |
+
state the exclusion of warranty; and each file should have at least
|
| 632 |
+
the "copyright" line and a pointer to where the full notice is found.
|
| 633 |
+
|
| 634 |
+
<one line to give the program's name and a brief idea of what it does.>
|
| 635 |
+
Copyright (C) <year> <name of author>
|
| 636 |
+
|
| 637 |
+
This program is free software: you can redistribute it and/or modify
|
| 638 |
+
it under the terms of the GNU General Public License as published by
|
| 639 |
+
the Free Software Foundation, either version 3 of the License, or
|
| 640 |
+
(at your option) any later version.
|
| 641 |
+
|
| 642 |
+
This program is distributed in the hope that it will be useful,
|
| 643 |
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
| 644 |
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
| 645 |
+
GNU General Public License for more details.
|
| 646 |
+
|
| 647 |
+
You should have received a copy of the GNU General Public License
|
| 648 |
+
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
| 649 |
+
|
| 650 |
+
Also add information on how to contact you by electronic and paper mail.
|
| 651 |
+
|
| 652 |
+
If the program does terminal interaction, make it output a short
|
| 653 |
+
notice like this when it starts in an interactive mode:
|
| 654 |
+
|
| 655 |
+
<program> Copyright (C) <year> <name of author>
|
| 656 |
+
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
| 657 |
+
This is free software, and you are welcome to redistribute it
|
| 658 |
+
under certain conditions; type `show c' for details.
|
| 659 |
+
|
| 660 |
+
The hypothetical commands `show w' and `show c' should show the appropriate
|
| 661 |
+
parts of the General Public License. Of course, your program's commands
|
| 662 |
+
might be different; for a GUI interface, you would use an "about box".
|
| 663 |
+
|
| 664 |
+
You should also get your employer (if you work as a programmer) or school,
|
| 665 |
+
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
| 666 |
+
For more information on this, and how to apply and follow the GNU GPL, see
|
| 667 |
+
<https://www.gnu.org/licenses/>.
|
| 668 |
+
|
| 669 |
+
The GNU General Public License does not permit incorporating your program
|
| 670 |
+
into proprietary programs. If your program is a subroutine library, you
|
| 671 |
+
may consider it more useful to permit linking proprietary applications with
|
| 672 |
+
the library. If this is what you want to do, use the GNU Lesser General
|
| 673 |
+
Public License instead of this License. But first, please read
|
| 674 |
+
<https://www.gnu.org/licenses/why-not-lgpl.html>.
|
MANIFEST.in
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
include LICENSE
|
| 2 |
+
include README.md
|
| 3 |
+
global-exclude *.py[cod] __pycache__
|
| 4 |
+
prune temp
|
| 5 |
+
prune tmp
|
| 6 |
+
prune downloads
|
| 7 |
+
prune hardsubs_m
|
| 8 |
+
prune softsubs_m
|
| 9 |
+
prune m
|
app.py
CHANGED
|
@@ -7,6 +7,8 @@ import threading
|
|
| 7 |
import signal
|
| 8 |
import sys
|
| 9 |
import re
|
|
|
|
|
|
|
| 10 |
from typing import Optional, List
|
| 11 |
from i18n.i18n import I18nAuto
|
| 12 |
from header import badges, description
|
|
@@ -14,6 +16,8 @@ i18n = I18nAuto()
|
|
| 14 |
|
| 15 |
# Variável global para armazenar o processo atual
|
| 16 |
current_process: Optional[subprocess.Popen] = None
|
|
|
|
|
|
|
| 17 |
|
| 18 |
# Força o uso de UTF-8 para o Python
|
| 19 |
os.environ["PYTHONIOENCODING"] = "utf-8"
|
|
@@ -66,15 +70,141 @@ def stop_process():
|
|
| 66 |
return "Process stopped by user"
|
| 67 |
return "No process running"
|
| 68 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
def run_legen(
|
| 70 |
input_path,
|
|
|
|
| 71 |
transcription_engine,
|
| 72 |
transcription_model,
|
| 73 |
compute_type,
|
| 74 |
device,
|
| 75 |
batch_size,
|
|
|
|
| 76 |
input_lang,
|
| 77 |
translate_lang,
|
|
|
|
|
|
|
|
|
|
| 78 |
video_codec,
|
| 79 |
audio_codec,
|
| 80 |
output_softsubs_path,
|
|
@@ -85,46 +215,90 @@ def run_legen(
|
|
| 85 |
disable_srt,
|
| 86 |
disable_softsubs,
|
| 87 |
disable_hardsubs,
|
|
|
|
|
|
|
| 88 |
progress=gr.Progress()
|
| 89 |
):
|
| 90 |
global current_process
|
|
|
|
|
|
|
| 91 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
if not input_path:
|
| 93 |
-
return "Please provide an input path"
|
| 94 |
|
| 95 |
-
if not os.path.exists(input_path):
|
| 96 |
-
return "Input path does not exist"
|
| 97 |
|
| 98 |
if not os.path.exists("legen.py"):
|
| 99 |
return "legen.py not found in current directory"
|
| 100 |
|
| 101 |
-
cmd = [
|
| 102 |
|
| 103 |
-
#
|
| 104 |
if normalize: cmd.append("--norm")
|
| 105 |
if overwrite: cmd.append("--overwrite")
|
| 106 |
if copy_files: cmd.append("--copy_files")
|
| 107 |
if disable_srt: cmd.append("--disable_srt")
|
| 108 |
if disable_softsubs: cmd.append("--disable_softsubs")
|
| 109 |
if disable_hardsubs: cmd.append("--disable_hardsubs")
|
|
|
|
|
|
|
|
|
|
| 110 |
|
| 111 |
-
#
|
| 112 |
cmd.extend(["-ts:e", transcription_engine])
|
| 113 |
cmd.extend(["-ts:m", transcription_model])
|
| 114 |
cmd.extend(["-ts:d", device])
|
| 115 |
cmd.extend(["-ts:c", compute_type])
|
| 116 |
cmd.extend(["-ts:b", str(batch_size)])
|
|
|
|
| 117 |
|
| 118 |
if translate_lang != "none":
|
| 119 |
cmd.extend(["--translate", translate_lang])
|
|
|
|
|
|
|
| 120 |
if input_lang != "auto":
|
| 121 |
cmd.extend(["--input_lang", input_lang])
|
| 122 |
|
| 123 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
cmd.extend(["-c:v", video_codec])
|
| 125 |
cmd.extend(["-c:a", audio_codec])
|
| 126 |
|
| 127 |
-
#
|
| 128 |
if output_softsubs_path:
|
| 129 |
cmd.extend(["-o:s", output_softsubs_path])
|
| 130 |
if output_hardsubs_path:
|
|
@@ -139,7 +313,7 @@ def run_legen(
|
|
| 139 |
current_process = subprocess.Popen(
|
| 140 |
cmd,
|
| 141 |
stdout=subprocess.PIPE,
|
| 142 |
-
stderr=subprocess.STDOUT,
|
| 143 |
text=True,
|
| 144 |
bufsize=1,
|
| 145 |
universal_newlines=True,
|
|
@@ -151,6 +325,10 @@ def run_legen(
|
|
| 151 |
|
| 152 |
output_lines: List[str] = []
|
| 153 |
last_progress_update = 0
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
|
| 155 |
while True:
|
| 156 |
line = current_process.stdout.readline()
|
|
@@ -163,131 +341,259 @@ def run_legen(
|
|
| 163 |
clean_line = process_output(line, progress)
|
| 164 |
output_lines.append(clean_line)
|
| 165 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
# Atualiza a saída periodicamente
|
| 167 |
if len(output_lines) - last_progress_update >= 5:
|
| 168 |
-
yield "\n".join(output_lines)
|
| 169 |
last_progress_update = len(output_lines)
|
| 170 |
|
| 171 |
except Exception as e:
|
| 172 |
output_lines.append(f"Error processing output: {str(e)}")
|
| 173 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
if current_process.poll() == 0:
|
| 175 |
final_output = "Processing completed successfully!\n\n" + "\n".join(output_lines)
|
| 176 |
else:
|
| 177 |
final_output = f"Process ended with error code {current_process.poll()}\n\n" + "\n".join(output_lines)
|
| 178 |
|
| 179 |
current_process = None
|
| 180 |
-
|
| 181 |
|
| 182 |
except Exception as e:
|
| 183 |
current_process = None
|
| 184 |
-
|
|
|
|
|
|
|
| 185 |
|
| 186 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
gr.Markdown(badges)
|
| 188 |
gr.Markdown(description)
|
| 189 |
-
title="LeGen"
|
| 190 |
|
| 191 |
with gr.Row():
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
with gr.
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
label=i18n("Batch Size"),
|
| 224 |
-
precision=0
|
| 225 |
-
)
|
| 226 |
-
with gr.Row():
|
| 227 |
-
input_lang = gr.Dropdown(
|
| 228 |
-
choices=["auto", "en", "es", "pt", "fr", "de", "it", "ja", "ko", "zh"],
|
| 229 |
-
value="auto",
|
| 230 |
-
label=i18n("Input Language")
|
| 231 |
-
)
|
| 232 |
-
translate_lang = gr.Dropdown(
|
| 233 |
-
choices=["none", "en", "es", "pt", "fr", "de", "it", "ja", "ko", "zh"],
|
| 234 |
-
value="none",
|
| 235 |
-
label=i18n("Translate to")
|
| 236 |
-
)
|
| 237 |
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
)
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
)
|
| 256 |
-
output_hardsubs_path = gr.Textbox(
|
| 257 |
-
value="hardsubs_output",
|
| 258 |
-
label=i18n("Hardsubs Output Path")
|
| 259 |
-
)
|
| 260 |
-
|
| 261 |
-
# Options
|
| 262 |
-
with gr.Group():
|
| 263 |
-
with gr.Row():
|
| 264 |
-
normalize = gr.Checkbox(label=i18n("Normalize folder times"), value=False)
|
| 265 |
-
overwrite = gr.Checkbox(label=i18n("Overwrite existing files"), value=False)
|
| 266 |
-
copy_files = gr.Checkbox(label=i18n("Copy non-video files"), value=False)
|
| 267 |
-
with gr.Row():
|
| 268 |
-
disable_srt = gr.Checkbox(label=i18n("Disable SRT generation"), value=False)
|
| 269 |
-
disable_softsubs = gr.Checkbox(label=i18n("Disable softsubs"), value=False)
|
| 270 |
-
disable_hardsubs = gr.Checkbox(label=i18n("Disable hardsubs"), value=False)
|
| 271 |
-
|
| 272 |
-
# Run Button, Stop Button and Output
|
| 273 |
-
with gr.Row():
|
| 274 |
-
with gr.Column():
|
| 275 |
-
run_btn = gr.Button(i18n("Run LeGen"), variant="primary")
|
| 276 |
-
stop_btn = gr.Button(i18n("Stop"), variant="stop")
|
| 277 |
-
output = gr.Textbox(label=i18n("Output"), lines=5)
|
| 278 |
|
| 279 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 280 |
run_btn.click(
|
| 281 |
fn=run_legen,
|
| 282 |
inputs=[
|
| 283 |
input_path,
|
|
|
|
| 284 |
transcription_engine,
|
| 285 |
transcription_model,
|
| 286 |
compute_type,
|
| 287 |
device,
|
| 288 |
batch_size,
|
|
|
|
| 289 |
input_lang,
|
| 290 |
translate_lang,
|
|
|
|
|
|
|
|
|
|
| 291 |
video_codec,
|
| 292 |
audio_codec,
|
| 293 |
output_softsubs_path,
|
|
@@ -297,19 +603,39 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
|
|
| 297 |
copy_files,
|
| 298 |
disable_srt,
|
| 299 |
disable_softsubs,
|
| 300 |
-
disable_hardsubs
|
|
|
|
|
|
|
| 301 |
],
|
| 302 |
-
outputs=output
|
| 303 |
)
|
| 304 |
|
| 305 |
-
stop_btn.click(
|
| 306 |
-
|
| 307 |
-
inputs=[],
|
| 308 |
-
outputs=output
|
| 309 |
-
)
|
| 310 |
gr.Markdown("""
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
|
|
|
|
|
|
| 314 |
if __name__ == "__main__":
|
| 315 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
import signal
|
| 8 |
import sys
|
| 9 |
import re
|
| 10 |
+
import shutil
|
| 11 |
+
import argparse
|
| 12 |
from typing import Optional, List
|
| 13 |
from i18n.i18n import I18nAuto
|
| 14 |
from header import badges, description
|
|
|
|
| 16 |
|
| 17 |
# Variável global para armazenar o processo atual
|
| 18 |
current_process: Optional[subprocess.Popen] = None
|
| 19 |
+
# Flag global para ambiente HuggingFace
|
| 20 |
+
IS_HF = False
|
| 21 |
|
| 22 |
# Força o uso de UTF-8 para o Python
|
| 23 |
os.environ["PYTHONIOENCODING"] = "utf-8"
|
|
|
|
| 70 |
return "Process stopped by user"
|
| 71 |
return "No process running"
|
| 72 |
|
| 73 |
+
def find_new_files(directories, start_time):
|
| 74 |
+
new_files = []
|
| 75 |
+
for directory in directories:
|
| 76 |
+
dir_path = Path(directory)
|
| 77 |
+
if dir_path.exists():
|
| 78 |
+
for file_path in dir_path.rglob("*"):
|
| 79 |
+
if file_path.is_file() and file_path.stat().st_mtime > start_time:
|
| 80 |
+
new_files.append(str(file_path))
|
| 81 |
+
return new_files
|
| 82 |
+
|
| 83 |
+
def prepare_display_files(file_paths):
|
| 84 |
+
"""
|
| 85 |
+
Copies files to a temp directory with names that include their parent folder
|
| 86 |
+
for better identification in the UI.
|
| 87 |
+
Example: hardsubs_output/video.mp4 -> [hardsubs_output] video.mp4
|
| 88 |
+
"""
|
| 89 |
+
if not file_paths:
|
| 90 |
+
return []
|
| 91 |
+
|
| 92 |
+
import tempfile
|
| 93 |
+
|
| 94 |
+
# Create a unique temp directory for this batch request to avoid collisions
|
| 95 |
+
temp_dir = Path(tempfile.mkdtemp(prefix="legen_display_"))
|
| 96 |
+
|
| 97 |
+
new_paths = []
|
| 98 |
+
root_path = Path(".").resolve()
|
| 99 |
+
|
| 100 |
+
for file_path in file_paths:
|
| 101 |
+
try:
|
| 102 |
+
path_obj = Path(file_path).resolve()
|
| 103 |
+
|
| 104 |
+
# Try to get relative path to project root
|
| 105 |
+
try:
|
| 106 |
+
rel_path = path_obj.relative_to(root_path)
|
| 107 |
+
parts = rel_path.parts
|
| 108 |
+
|
| 109 |
+
# If it's in a subfolder, prefix with that folder name
|
| 110 |
+
# e.g. "input/file.mp4" -> "[input] file.mp4"
|
| 111 |
+
# "hardsubs/file.mp4" -> "[hardsubs] file.mp4"
|
| 112 |
+
if len(parts) > 1:
|
| 113 |
+
parent_str = parts[0]
|
| 114 |
+
filename = parts[-1]
|
| 115 |
+
# Clean brackets if they already exist to avoid [[...]]
|
| 116 |
+
new_name = f"[{parent_str}] {filename}"
|
| 117 |
+
else:
|
| 118 |
+
new_name = path_obj.name
|
| 119 |
+
|
| 120 |
+
except ValueError:
|
| 121 |
+
# File is outside root
|
| 122 |
+
new_name = f"[{path_obj.parent.name}] {path_obj.name}"
|
| 123 |
+
|
| 124 |
+
dest_file = temp_dir / new_name
|
| 125 |
+
shutil.copy2(path_obj, dest_file)
|
| 126 |
+
new_paths.append(str(dest_file))
|
| 127 |
+
|
| 128 |
+
except Exception as e:
|
| 129 |
+
print(f"Error preparing file {file_path}: {e}")
|
| 130 |
+
new_paths.append(file_path)
|
| 131 |
+
|
| 132 |
+
return new_paths
|
| 133 |
+
|
| 134 |
+
def get_projects():
|
| 135 |
+
"""List projects based on files in input directory."""
|
| 136 |
+
input_dir = Path("input")
|
| 137 |
+
if not input_dir.exists():
|
| 138 |
+
return []
|
| 139 |
+
|
| 140 |
+
projects = []
|
| 141 |
+
for f in input_dir.glob("*"):
|
| 142 |
+
if f.is_file():
|
| 143 |
+
# Remove timestamp like _1738... if present to get clean project name
|
| 144 |
+
# Or just use full filename as project id
|
| 145 |
+
projects.append(f.name)
|
| 146 |
+
return sorted(projects, reverse=True)
|
| 147 |
+
|
| 148 |
+
def get_project_files(project_filename):
|
| 149 |
+
"""Get all files related to a project filename with better discovery."""
|
| 150 |
+
if not project_filename:
|
| 151 |
+
return []
|
| 152 |
+
|
| 153 |
+
print(f"Searching files for project: {project_filename}")
|
| 154 |
+
files = []
|
| 155 |
+
|
| 156 |
+
# 1. Input file
|
| 157 |
+
try:
|
| 158 |
+
input_path = (Path("input") / project_filename).resolve()
|
| 159 |
+
if input_path.exists():
|
| 160 |
+
files.append(str(input_path))
|
| 161 |
+
except Exception as e:
|
| 162 |
+
print(f"Error finding input file: {e}")
|
| 163 |
+
|
| 164 |
+
# Base stem for matching
|
| 165 |
+
stem = Path(project_filename).stem
|
| 166 |
+
|
| 167 |
+
# Directories to scan: outputs and root (for summaries)
|
| 168 |
+
dirs_to_scan = ["softsubs_output", "hardsubs_output", "."]
|
| 169 |
+
|
| 170 |
+
for d in dirs_to_scan:
|
| 171 |
+
try:
|
| 172 |
+
dir_path = Path(d).resolve()
|
| 173 |
+
if dir_path.exists():
|
| 174 |
+
# Use glob to scan the directory
|
| 175 |
+
for f in dir_path.glob("*"):
|
| 176 |
+
if f.is_file():
|
| 177 |
+
# Match if filename starts with the project stem
|
| 178 |
+
# Avoid duplicating the input file if scanning input dir (not in list but safety check)
|
| 179 |
+
if f.name.startswith(stem) and f.resolve() != input_path:
|
| 180 |
+
files.append(str(f.resolve()))
|
| 181 |
+
except Exception as e:
|
| 182 |
+
print(f"Error scanning directory {d}: {e}")
|
| 183 |
+
|
| 184 |
+
unique_files = sorted(list(set(files)))
|
| 185 |
+
print(f"Found {len(unique_files)} related files.")
|
| 186 |
+
|
| 187 |
+
# Return processed files with friendly names
|
| 188 |
+
return prepare_display_files(unique_files)
|
| 189 |
+
|
| 190 |
+
def refresh_projects():
|
| 191 |
+
return gr.update(choices=get_projects())
|
| 192 |
+
|
| 193 |
+
|
| 194 |
def run_legen(
|
| 195 |
input_path,
|
| 196 |
+
input_file,
|
| 197 |
transcription_engine,
|
| 198 |
transcription_model,
|
| 199 |
compute_type,
|
| 200 |
device,
|
| 201 |
batch_size,
|
| 202 |
+
transcription_vad,
|
| 203 |
input_lang,
|
| 204 |
translate_lang,
|
| 205 |
+
translate_engine,
|
| 206 |
+
gemini_api_key,
|
| 207 |
+
tltw,
|
| 208 |
video_codec,
|
| 209 |
audio_codec,
|
| 210 |
output_softsubs_path,
|
|
|
|
| 215 |
disable_srt,
|
| 216 |
disable_softsubs,
|
| 217 |
disable_hardsubs,
|
| 218 |
+
download_remote_subs,
|
| 219 |
+
process_input_subs,
|
| 220 |
progress=gr.Progress()
|
| 221 |
):
|
| 222 |
global current_process
|
| 223 |
+
import time
|
| 224 |
+
start_time = time.time()
|
| 225 |
|
| 226 |
+
# Handle File Upload
|
| 227 |
+
if input_file is not None:
|
| 228 |
+
try:
|
| 229 |
+
input_dir = Path("input")
|
| 230 |
+
input_dir.mkdir(exist_ok=True)
|
| 231 |
+
|
| 232 |
+
original_name = os.path.basename(input_file)
|
| 233 |
+
|
| 234 |
+
# Use timestamp to avoid cache/overwrite issues ONLY on HF Spaces
|
| 235 |
+
if IS_HF:
|
| 236 |
+
import time
|
| 237 |
+
timestamp = int(time.time())
|
| 238 |
+
name, ext = os.path.splitext(original_name)
|
| 239 |
+
new_filename = f"{name}_{timestamp}{ext}"
|
| 240 |
+
else:
|
| 241 |
+
new_filename = original_name
|
| 242 |
+
|
| 243 |
+
dest_path = input_dir / new_filename
|
| 244 |
+
# Copy file to input directory
|
| 245 |
+
shutil.copy2(input_file, dest_path)
|
| 246 |
+
# Use the new file path as input
|
| 247 |
+
input_path = str(dest_path)
|
| 248 |
+
|
| 249 |
+
# Update the output to inform user (optional but helpful)
|
| 250 |
+
if IS_HF:
|
| 251 |
+
print(f"File uploaded and saved as: {input_path}")
|
| 252 |
+
|
| 253 |
+
except Exception as e:
|
| 254 |
+
return f"Error processing uploaded file: {str(e)}"
|
| 255 |
+
|
| 256 |
if not input_path:
|
| 257 |
+
return "Please provide an input path or upload a file"
|
| 258 |
|
|
|
|
|
|
|
| 259 |
|
| 260 |
if not os.path.exists("legen.py"):
|
| 261 |
return "legen.py not found in current directory"
|
| 262 |
|
| 263 |
+
cmd = [sys.executable, "legen.py", "-i", input_path]
|
| 264 |
|
| 265 |
+
# Flags
|
| 266 |
if normalize: cmd.append("--norm")
|
| 267 |
if overwrite: cmd.append("--overwrite")
|
| 268 |
if copy_files: cmd.append("--copy_files")
|
| 269 |
if disable_srt: cmd.append("--disable_srt")
|
| 270 |
if disable_softsubs: cmd.append("--disable_softsubs")
|
| 271 |
if disable_hardsubs: cmd.append("--disable_hardsubs")
|
| 272 |
+
if download_remote_subs: cmd.append("-dl:rs")
|
| 273 |
+
if process_input_subs: cmd.append("--process_input_subs")
|
| 274 |
+
if tltw: cmd.append("--tltw")
|
| 275 |
|
| 276 |
+
# Transcription settings
|
| 277 |
cmd.extend(["-ts:e", transcription_engine])
|
| 278 |
cmd.extend(["-ts:m", transcription_model])
|
| 279 |
cmd.extend(["-ts:d", device])
|
| 280 |
cmd.extend(["-ts:c", compute_type])
|
| 281 |
cmd.extend(["-ts:b", str(batch_size)])
|
| 282 |
+
cmd.extend(["-ts:v", transcription_vad])
|
| 283 |
|
| 284 |
if translate_lang != "none":
|
| 285 |
cmd.extend(["--translate", translate_lang])
|
| 286 |
+
cmd.extend(["--translate_engine", translate_engine])
|
| 287 |
+
|
| 288 |
if input_lang != "auto":
|
| 289 |
cmd.extend(["--input_lang", input_lang])
|
| 290 |
|
| 291 |
+
# Gemini Keys
|
| 292 |
+
if gemini_api_key:
|
| 293 |
+
keys = [k.strip() for k in gemini_api_key.replace('\n', ',').split(',') if k.strip()]
|
| 294 |
+
for key in keys:
|
| 295 |
+
cmd.extend(["--gemini_api_key", key])
|
| 296 |
+
|
| 297 |
+
# Codecs
|
| 298 |
cmd.extend(["-c:v", video_codec])
|
| 299 |
cmd.extend(["-c:a", audio_codec])
|
| 300 |
|
| 301 |
+
# Paths
|
| 302 |
if output_softsubs_path:
|
| 303 |
cmd.extend(["-o:s", output_softsubs_path])
|
| 304 |
if output_hardsubs_path:
|
|
|
|
| 313 |
current_process = subprocess.Popen(
|
| 314 |
cmd,
|
| 315 |
stdout=subprocess.PIPE,
|
| 316 |
+
stderr=subprocess.STDOUT,
|
| 317 |
text=True,
|
| 318 |
bufsize=1,
|
| 319 |
universal_newlines=True,
|
|
|
|
| 325 |
|
| 326 |
output_lines: List[str] = []
|
| 327 |
last_progress_update = 0
|
| 328 |
+
current_status_text = "⏳ Iniciando..."
|
| 329 |
+
|
| 330 |
+
# Initial yield to disable button
|
| 331 |
+
yield "\n".join(output_lines), gr.update(value=current_status_text, interactive=False), None
|
| 332 |
|
| 333 |
while True:
|
| 334 |
line = current_process.stdout.readline()
|
|
|
|
| 341 |
clean_line = process_output(line, progress)
|
| 342 |
output_lines.append(clean_line)
|
| 343 |
|
| 344 |
+
# Detect status changes
|
| 345 |
+
lower_line = clean_line.lower()
|
| 346 |
+
if "transcribing" in lower_line:
|
| 347 |
+
current_status_text = "🎙️ Transcrevendo..."
|
| 348 |
+
elif "translating" in lower_line:
|
| 349 |
+
current_status_text = "🌍 Traduzindo..."
|
| 350 |
+
elif "generating tltw" in lower_line:
|
| 351 |
+
current_status_text = "🤖 Gerando Resumo..."
|
| 352 |
+
elif "inserting subtitle" in lower_line or "burning" in lower_line:
|
| 353 |
+
current_status_text = "🎬 Processando Vídeo..."
|
| 354 |
+
elif "downloading" in lower_line:
|
| 355 |
+
current_status_text = "⬇️ Baixando..."
|
| 356 |
+
elif "running vidqa" in lower_line:
|
| 357 |
+
current_status_text = "🔍 Verificando..."
|
| 358 |
+
|
| 359 |
# Atualiza a saída periodicamente
|
| 360 |
if len(output_lines) - last_progress_update >= 5:
|
| 361 |
+
yield "\n".join(output_lines), gr.update(value=current_status_text), None
|
| 362 |
last_progress_update = len(output_lines)
|
| 363 |
|
| 364 |
except Exception as e:
|
| 365 |
output_lines.append(f"Error processing output: {str(e)}")
|
| 366 |
|
| 367 |
+
# Process finished, look for files
|
| 368 |
+
raw_found_files = find_new_files([output_softsubs_path or "softsubs_output", output_hardsubs_path or "hardsubs_output"], start_time)
|
| 369 |
+
found_files = prepare_display_files(raw_found_files)
|
| 370 |
+
|
| 371 |
if current_process.poll() == 0:
|
| 372 |
final_output = "Processing completed successfully!\n\n" + "\n".join(output_lines)
|
| 373 |
else:
|
| 374 |
final_output = f"Process ended with error code {current_process.poll()}\n\n" + "\n".join(output_lines)
|
| 375 |
|
| 376 |
current_process = None
|
| 377 |
+
yield final_output, gr.update(value="EXECUTAR LEGEN", interactive=True), found_files
|
| 378 |
|
| 379 |
except Exception as e:
|
| 380 |
current_process = None
|
| 381 |
+
yield f"Error: {str(e)}", gr.update(value="EXECUTAR LEGEN", interactive=True), None
|
| 382 |
+
|
| 383 |
+
# --- UI Configuration ---
|
| 384 |
|
| 385 |
+
custom_css = """
|
| 386 |
+
footer {display:none !important}
|
| 387 |
+
.gradio-container {width: 98% !important; max-width: 100% !important; margin: auto;}
|
| 388 |
+
#run_btn {background: linear-gradient(90deg, #4CAF50 0%, #45a049 100%); color: white; border: none; font-weight: bold; font-size: 18px; padding: 12px;}
|
| 389 |
+
#run_btn:hover {box-shadow: 0 4px 15px rgba(76, 175, 80, 0.4);}
|
| 390 |
+
"""
|
| 391 |
+
|
| 392 |
+
with gr.Blocks() as demo:
|
| 393 |
gr.Markdown(badges)
|
| 394 |
gr.Markdown(description)
|
|
|
|
| 395 |
|
| 396 |
with gr.Row():
|
| 397 |
+
# Left Column - Controls (Scale 3 for better spacing on large screens)
|
| 398 |
+
with gr.Column(scale=3):
|
| 399 |
+
with gr.Tabs():
|
| 400 |
+
|
| 401 |
+
# --- Tab 1: Arquivos ---
|
| 402 |
+
with gr.TabItem("📁 Step 1: Arquivos"):
|
| 403 |
+
gr.Markdown("### 📂 Configuração de Entrada e Saída")
|
| 404 |
+
with gr.Group():
|
| 405 |
+
input_path = gr.Textbox(
|
| 406 |
+
label=i18n("Input Path"),
|
| 407 |
+
placeholder=i18n("Caminho do arquivo, pasta ou URL") if not IS_HF else "Input manual bloqueado (Use Upload acima)",
|
| 408 |
+
info=i18n("Cole o caminho local ou uma URL (YouTube, etc).") if not IS_HF else "Entrada manual desativada por segurança no ambiente HF.",
|
| 409 |
+
interactive=not IS_HF
|
| 410 |
+
)
|
| 411 |
+
input_file = gr.File(
|
| 412 |
+
label=i18n("Upload de Arquivo"),
|
| 413 |
+
type="filepath",
|
| 414 |
+
file_count="single",
|
| 415 |
+
height=300
|
| 416 |
+
)
|
| 417 |
+
|
| 418 |
+
with gr.Accordion("Configurações de Saída (Opcional)", open=False):
|
| 419 |
+
with gr.Group():
|
| 420 |
+
output_softsubs_path = gr.Textbox(value="softsubs_output", label=i18n("Pasta Softsubs (Legendas)"))
|
| 421 |
+
output_hardsubs_path = gr.Textbox(value="hardsubs_output", label=i18n("Pasta Hardsubs (Queimadas)"))
|
| 422 |
+
|
| 423 |
+
download_remote_subs = gr.Checkbox(
|
| 424 |
+
label=i18n("Baixar Legendas Remotas (para URLs)"),
|
| 425 |
+
value=False,
|
| 426 |
+
info=i18n("Tenta baixar legendas oficiais se disponíveis na URL.")
|
| 427 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 428 |
|
| 429 |
+
# --- Tab 2: Inteligência ---
|
| 430 |
+
with gr.TabItem("🧠 Step 2: Inteligência"):
|
| 431 |
+
gr.Markdown("### 🎙️ Configuração de Transcrição")
|
| 432 |
+
with gr.Group():
|
| 433 |
+
transcription_engine = gr.Dropdown(
|
| 434 |
+
choices=["whisperx", "whisper"],
|
| 435 |
+
value="whisperx",
|
| 436 |
+
label=i18n("Engine de Transcrição"),
|
| 437 |
+
info=i18n("WhisperX é geralmente mais rápido e preciso para alinhamento.")
|
| 438 |
+
)
|
| 439 |
+
with gr.Row():
|
| 440 |
+
transcription_model = gr.Dropdown(
|
| 441 |
+
choices=["tiny", "base", "small", "medium", "large", "large-v1", "large-v2", "large-v3", "large-v3-turbo",],
|
| 442 |
+
value="large-v3",
|
| 443 |
+
label=i18n("Modelo"),
|
| 444 |
+
info=i18n("Modelos maiores (large) são mais precisos, mas exigem mais VRAM.")
|
| 445 |
+
)
|
| 446 |
+
compute_type = gr.Dropdown(
|
| 447 |
+
choices=["auto", "int8", "float16", "float32"],
|
| 448 |
+
value="auto",
|
| 449 |
+
label=i18n("Tipo de Computação"),
|
| 450 |
+
info=i18n("int8 economiza memória, float16 é padrão para GPU.")
|
| 451 |
+
)
|
| 452 |
+
with gr.Row():
|
| 453 |
+
device = gr.Dropdown(choices=["auto", "cpu", "cuda"], value="auto", label=i18n("Dispositivo/Device"))
|
| 454 |
+
batch_size = gr.Number(value=4, label=i18n("Batch Size"), precision=0)
|
| 455 |
+
transcription_vad = gr.Dropdown(
|
| 456 |
+
choices=["silero", "pyannote"],
|
| 457 |
+
value="silero",
|
| 458 |
+
label=i18n("Método VAD"),
|
| 459 |
+
info=i18n("Detector de atividade de voz.")
|
| 460 |
+
)
|
| 461 |
+
input_lang = gr.Dropdown(
|
| 462 |
+
choices=["auto", "en", "es", "pt", "fr", "de", "it", "ja", "ko", "zh"],
|
| 463 |
+
value="auto",
|
| 464 |
+
label=i18n("Idioma do Áudio Original")
|
| 465 |
+
)
|
| 466 |
+
|
| 467 |
+
gr.Markdown("### 🤖 Recursos IA & Gemini")
|
| 468 |
+
with gr.Group():
|
| 469 |
+
gemini_api_key = gr.Textbox(
|
| 470 |
+
label=i18n("🔑 Gemini API Key"),
|
| 471 |
+
placeholder=i18n("Cole sua API Key aqui"),
|
| 472 |
+
type="password",
|
| 473 |
+
info=i18n("Necessário para tradução Gemini e/ou Resumo TLTW.")
|
| 474 |
+
)
|
| 475 |
+
show_api_key = gr.Checkbox(label="Mostrar API Key", value=False)
|
| 476 |
+
|
| 477 |
+
def toggle_api_key_vis(show):
|
| 478 |
+
return gr.update(type="text" if show else "password")
|
| 479 |
+
|
| 480 |
+
show_api_key.change(fn=toggle_api_key_vis, inputs=show_api_key, outputs=gemini_api_key)
|
| 481 |
+
|
| 482 |
+
tltw = gr.Checkbox(
|
| 483 |
+
label=i18n("Gerar Resumo TLTW (Too Long To Watch)"),
|
| 484 |
+
value=False,
|
| 485 |
+
info=i18n("Gera um resumo do conteúdo usando Gemini.")
|
| 486 |
+
)
|
| 487 |
+
|
| 488 |
+
gr.Markdown("### 🌍 Tradução")
|
| 489 |
+
with gr.Group():
|
| 490 |
+
translate_lang = gr.Dropdown(
|
| 491 |
+
choices=["none", "en", "es", "pt", "fr", "de", "it", "ja", "ko", "zh"],
|
| 492 |
+
value="none",
|
| 493 |
+
label=i18n("Traduzir para"),
|
| 494 |
+
info=i18n("Selecione 'none' para apenas transcrever.")
|
| 495 |
+
)
|
| 496 |
+
|
| 497 |
+
# Conditional Inputs
|
| 498 |
+
with gr.Column(visible=False) as translation_options_box:
|
| 499 |
+
translate_engine = gr.Dropdown(
|
| 500 |
+
choices=["google", "gemini"],
|
| 501 |
+
value="google",
|
| 502 |
+
label=i18n("Motor de Tradução"),
|
| 503 |
+
interactive=True
|
| 504 |
+
)
|
| 505 |
+
|
| 506 |
+
# Event Logic for Visibility
|
| 507 |
+
def update_trans_vis(lang):
|
| 508 |
+
if lang == "none":
|
| 509 |
+
return gr.update(visible=False)
|
| 510 |
+
else:
|
| 511 |
+
return gr.update(visible=True)
|
| 512 |
+
|
| 513 |
+
translate_lang.change(fn=update_trans_vis, inputs=translate_lang, outputs=translation_options_box)
|
| 514 |
+
|
| 515 |
+
# --- Tab 3: Avançado ---
|
| 516 |
+
with gr.TabItem("⚙️ Step 3: Exportação"):
|
| 517 |
+
gr.Markdown("### 🎬 Codecs")
|
| 518 |
+
with gr.Group():
|
| 519 |
+
with gr.Row():
|
| 520 |
+
video_codec = gr.Dropdown(
|
| 521 |
+
choices=["h264", "libx264", "h264_vaapi", "h264_nvenc", "hevc", "libx265", "hevc_vaapi"],
|
| 522 |
+
value="h264",
|
| 523 |
+
label=i18n("Codec de Vídeo")
|
| 524 |
+
)
|
| 525 |
+
audio_codec = gr.Dropdown(
|
| 526 |
+
choices=["aac", "libopus", "mp3", "vorbis"],
|
| 527 |
+
value="aac",
|
| 528 |
+
label=i18n("Codec de Áudio")
|
| 529 |
+
)
|
| 530 |
+
|
| 531 |
+
gr.Markdown("### 🔧 Opções de Processamento")
|
| 532 |
+
with gr.Row():
|
| 533 |
+
with gr.Column():
|
| 534 |
+
normalize = gr.Checkbox(label=i18n("Normalizar tempos"), value=False)
|
| 535 |
+
overwrite = gr.Checkbox(label=i18n("Sobrescrever arquivos"), value=False)
|
| 536 |
+
process_input_subs = gr.Checkbox(label=i18n("Processar .srt existentes"), value=False)
|
| 537 |
+
with gr.Column():
|
| 538 |
+
disable_srt = gr.Checkbox(label=i18n("Desativar geração .srt"), value=False)
|
| 539 |
+
disable_softsubs = gr.Checkbox(label=i18n("Desativar Softsubs (MP4)"), value=False)
|
| 540 |
+
disable_hardsubs = gr.Checkbox(label=i18n("Desativar Hardsubs (Queimadas)"), value=False)
|
| 541 |
+
with gr.Column():
|
| 542 |
+
copy_files = gr.Checkbox(label=i18n("Copiar arquivos extra"), value=False)
|
| 543 |
+
|
| 544 |
+
# --- Tab 4: Gerenciador de Arquivos ---
|
| 545 |
+
with gr.TabItem("🗃️ Gerenciador de Arquivos"):
|
| 546 |
+
gr.Markdown("### 📂 Meus Projetos e Arquivos")
|
| 547 |
+
with gr.Row():
|
| 548 |
+
with gr.Column(scale=4):
|
| 549 |
+
project_selector = gr.Dropdown(
|
| 550 |
+
label="Selecione um Projeto",
|
| 551 |
+
choices=get_projects(),
|
| 552 |
+
interactive=True,
|
| 553 |
+
info="Baseado nos arquivos na pasta 'input/'"
|
| 554 |
+
)
|
| 555 |
+
with gr.Column(scale=1):
|
| 556 |
+
refresh_btn = gr.Button("🔄 Atualizar Lista", size="sm")
|
| 557 |
+
|
| 558 |
+
project_files_display = gr.File(
|
| 559 |
+
label="Arquivos do Projeto",
|
| 560 |
+
file_count="multiple",
|
| 561 |
+
interactive=False,
|
| 562 |
+
height=400
|
| 563 |
)
|
| 564 |
+
|
| 565 |
+
project_selector.change(fn=get_project_files, inputs=project_selector, outputs=project_files_display)
|
| 566 |
+
refresh_btn.click(fn=refresh_projects, outputs=project_selector)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 567 |
|
| 568 |
+
|
| 569 |
+
# Right Column - Status & Actions (Scale 1)
|
| 570 |
+
with gr.Column(scale=1):
|
| 571 |
+
gr.Markdown("### 🚀 Status")
|
| 572 |
+
output = gr.Textbox(label=i18n("Log de Processamento"), lines=20, elem_id="output_box")
|
| 573 |
+
output_files = gr.File(label="📂 Arquivos Gerados", file_count="multiple", interactive=False, height=150)
|
| 574 |
+
|
| 575 |
+
with gr.Row():
|
| 576 |
+
run_btn = gr.Button(i18n("EXECUTAR LEGEN"), variant="primary", elem_id="run_btn", size="lg")
|
| 577 |
+
with gr.Row():
|
| 578 |
+
stop_btn = gr.Button(i18n("PARAR TUDO"), variant="stop", size="sm")
|
| 579 |
+
|
| 580 |
+
# Connect the run button
|
| 581 |
run_btn.click(
|
| 582 |
fn=run_legen,
|
| 583 |
inputs=[
|
| 584 |
input_path,
|
| 585 |
+
input_file,
|
| 586 |
transcription_engine,
|
| 587 |
transcription_model,
|
| 588 |
compute_type,
|
| 589 |
device,
|
| 590 |
batch_size,
|
| 591 |
+
transcription_vad,
|
| 592 |
input_lang,
|
| 593 |
translate_lang,
|
| 594 |
+
translate_engine,
|
| 595 |
+
gemini_api_key,
|
| 596 |
+
tltw,
|
| 597 |
video_codec,
|
| 598 |
audio_codec,
|
| 599 |
output_softsubs_path,
|
|
|
|
| 603 |
copy_files,
|
| 604 |
disable_srt,
|
| 605 |
disable_softsubs,
|
| 606 |
+
disable_hardsubs,
|
| 607 |
+
download_remote_subs,
|
| 608 |
+
process_input_subs
|
| 609 |
],
|
| 610 |
+
outputs=[output, run_btn, output_files]
|
| 611 |
)
|
| 612 |
|
| 613 |
+
stop_btn.click(fn=stop_process, inputs=[], outputs=output)
|
| 614 |
+
|
|
|
|
|
|
|
|
|
|
| 615 |
gr.Markdown("""
|
| 616 |
+
<div style="text-align: center; margin-top: 20px; opacity: 0.7;">
|
| 617 |
+
<small>WebUI por Rafa.Godoy | LeGen por MatheusBach</small>
|
| 618 |
+
</div>
|
| 619 |
+
""")
|
| 620 |
+
|
| 621 |
if __name__ == "__main__":
|
| 622 |
+
parser = argparse.ArgumentParser()
|
| 623 |
+
parser.add_argument("--colab", action="store_true", help="Optimize for Google Colab (share=True)")
|
| 624 |
+
parser.add_argument("--hf", action="store_true", help="Optimize for HuggingFace Spaces (timestamp uploads)")
|
| 625 |
+
args = parser.parse_args()
|
| 626 |
+
|
| 627 |
+
IS_HF = args.hf
|
| 628 |
+
|
| 629 |
+
launch_kwargs = {
|
| 630 |
+
"theme": gr.themes.Soft(text_size="lg", spacing_size="lg", radius_size="lg"),
|
| 631 |
+
"css": custom_css
|
| 632 |
+
}
|
| 633 |
+
|
| 634 |
+
if args.colab:
|
| 635 |
+
launch_kwargs["share"] = True
|
| 636 |
+
launch_kwargs["server_name"] = "0.0.0.0"
|
| 637 |
+
|
| 638 |
+
if args.hf:
|
| 639 |
+
launch_kwargs["server_name"] = "0.0.0.0"
|
| 640 |
+
|
| 641 |
+
demo.launch(**launch_kwargs)
|
debug_cpu.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
import sys
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
print("Debugging WhisperX on CPU...")
|
| 6 |
+
print(f"Python: {sys.version}")
|
| 7 |
+
|
| 8 |
+
try:
|
| 9 |
+
import torch
|
| 10 |
+
print(f"PyTorch version: {torch.__version__}")
|
| 11 |
+
except ImportError:
|
| 12 |
+
print("PyTorch not found!")
|
| 13 |
+
|
| 14 |
+
try:
|
| 15 |
+
import whisperx_legen_fork as whisperx
|
| 16 |
+
print("WhisperX module imported successfully.")
|
| 17 |
+
except ImportError as e:
|
| 18 |
+
print(f"Failed to import whisperx_legen_fork: {e}")
|
| 19 |
+
sys.exit(1)
|
| 20 |
+
|
| 21 |
+
device = "cpu"
|
| 22 |
+
compute_type = "float32" # Int8 or float32 are required for CPU
|
| 23 |
+
|
| 24 |
+
print(f"Attempting to load model 'tiny' on {device} with {compute_type}...")
|
| 25 |
+
|
| 26 |
+
try:
|
| 27 |
+
model = whisperx.load_model(
|
| 28 |
+
"tiny",
|
| 29 |
+
device=device,
|
| 30 |
+
compute_type=compute_type,
|
| 31 |
+
asr_options={"repetition_penalty": 1, "prompt_reset_on_temperature": 0.5, "no_repeat_ngram_size": 2}
|
| 32 |
+
)
|
| 33 |
+
print("SUCCESS: Model loaded correctly!")
|
| 34 |
+
except Exception as e:
|
| 35 |
+
print("\nERROR loading model:")
|
| 36 |
+
print(e)
|
| 37 |
+
import traceback
|
| 38 |
+
traceback.print_exc()
|
| 39 |
+
|
| 40 |
+
print("Done.")
|
device_utils.py
ADDED
|
@@ -0,0 +1,506 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Device detection utilities.
|
| 2 |
+
|
| 3 |
+
Provides a robust way to detect available accelerators (CUDA, MPS) and returns
|
| 4 |
+
useful metadata that can be used to pick the best compute backend.
|
| 5 |
+
|
| 6 |
+
When possible we rely on PyTorch for accurate information, falling back to
|
| 7 |
+
``nvidia-smi`` for a lightweight probe so that we can still inform the user
|
| 8 |
+
about available GPUs even when PyTorch is not ready to use them.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from dataclasses import dataclass, field
|
| 14 |
+
from contextlib import contextmanager
|
| 15 |
+
import importlib
|
| 16 |
+
import math
|
| 17 |
+
import shutil
|
| 18 |
+
import subprocess
|
| 19 |
+
import warnings
|
| 20 |
+
import sys
|
| 21 |
+
from typing import List, Optional, Tuple
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
_GIB = 1024 ** 3
|
| 25 |
+
|
| 26 |
+
_MODEL_VRAM_REQUIREMENTS_GB = {
|
| 27 |
+
"tiny": {"int8": 0.4, "int8_float16": 0.6, "float16": 1.0, "float32": 1.8},
|
| 28 |
+
"tiny.en": {"int8": 0.4, "int8_float16": 0.6, "float16": 1.0, "float32": 1.8},
|
| 29 |
+
"base": {"int8": 0.5, "int8_float16": 0.8, "float16": 1.1, "float32": 2.0},
|
| 30 |
+
"base.en": {"int8": 0.5, "int8_float16": 0.8, "float16": 1.1, "float32": 2.0},
|
| 31 |
+
"small": {"int8": 0.9, "int8_float16": 1.3, "float16": 2.0, "float32": 3.5},
|
| 32 |
+
"small.en": {"int8": 0.9, "int8_float16": 1.3, "float16": 2.0, "float32": 3.5},
|
| 33 |
+
"medium": {"int8": 2.2, "int8_float16": 3.0, "float16": 5.0, "float32": 9.0},
|
| 34 |
+
"medium.en": {"int8": 2.2, "int8_float16": 3.0, "float16": 5.0, "float32": 9.0},
|
| 35 |
+
"large": {"int8": 3.5, "int8_float16": 4.5, "float16": 10.0, "float32": 18.0},
|
| 36 |
+
"large-v1": {"int8": 3.5, "int8_float16": 4.5, "float16": 10.0, "float32": 18.0},
|
| 37 |
+
"large-v2": {"int8": 3.5, "int8_float16": 4.5, "float16": 10.0, "float32": 18.0},
|
| 38 |
+
"large-v3": {"int8": 3.5, "int8_float16": 4.5, "float16": 10.0, "float32": 18.0},
|
| 39 |
+
"large-v3-turbo": {"int8": 2.0, "int8_float16": 2.8, "float16": 6.0, "float32": 10.0},
|
| 40 |
+
"turbo": {"int8": 2.0, "int8_float16": 2.8, "float16": 6.0, "float32": 10.0},
|
| 41 |
+
"distil-large-v2": {"int8": 2.0, "int8_float16": 2.8, "float16": 6.0, "float32": 10.0},
|
| 42 |
+
"distil-medium.en": {"int8": 1.1, "int8_float16": 1.6, "float16": 3.0, "float32": 5.5},
|
| 43 |
+
"distil-small.en": {"int8": 0.5, "int8_float16": 0.8, "float16": 1.5, "float32": 2.5},
|
| 44 |
+
}
|
| 45 |
+
_DEFAULT_MODEL_VRAM_GB = {
|
| 46 |
+
"int8": 1.0,
|
| 47 |
+
"int8_float16": 1.5,
|
| 48 |
+
"float16": 6.0,
|
| 49 |
+
"float32": 10.0,
|
| 50 |
+
}
|
| 51 |
+
_FALLBACK_MODEL_VRAM_GB = 6.0
|
| 52 |
+
|
| 53 |
+
_GPU_ONLY_COMPUTE_TYPES = {"float16", "fp16", "bfloat16", "int8_float16", "int8_bfloat16"}
|
| 54 |
+
_FP16_COMPUTE_TYPES = {"float16", "fp16", "bfloat16", "int8_float16", "int8_bfloat16"}
|
| 55 |
+
|
| 56 |
+
_COMPUTE_CANONICAL = {
|
| 57 |
+
"int8": "int8",
|
| 58 |
+
"int8_float16": "int8_float16",
|
| 59 |
+
"int8_bfloat16": "int8_float16",
|
| 60 |
+
"int8_float32": "float32",
|
| 61 |
+
"float16": "float16",
|
| 62 |
+
"fp16": "float16",
|
| 63 |
+
"bfloat16": "float16",
|
| 64 |
+
"float32": "float32",
|
| 65 |
+
"int16": "float32",
|
| 66 |
+
"default": "float16",
|
| 67 |
+
"auto": "float16",
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
_TORCH_WARNINGS_CONFIGURED = False
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@dataclass
|
| 74 |
+
class DeviceInfo:
|
| 75 |
+
"""Structured information about the selected compute backend."""
|
| 76 |
+
|
| 77 |
+
backend: str
|
| 78 |
+
n_gpus: int = 0
|
| 79 |
+
gpu_names: List[str] = field(default_factory=list)
|
| 80 |
+
gpu_vram_bytes: List[int] = field(default_factory=list)
|
| 81 |
+
gpu_capabilities: List[Tuple[int, int]] = field(default_factory=list)
|
| 82 |
+
cuda_version: Optional[str] = None
|
| 83 |
+
driver_version: Optional[str] = None
|
| 84 |
+
messages: List[str] = field(default_factory=list)
|
| 85 |
+
issues: List[str] = field(default_factory=list)
|
| 86 |
+
notes: List[str] = field(default_factory=list)
|
| 87 |
+
resolved_compute_type: Optional[str] = None
|
| 88 |
+
selected_gpu_index: Optional[int] = None
|
| 89 |
+
|
| 90 |
+
def primary_gpu_name(self) -> Optional[str]:
|
| 91 |
+
if self.selected_gpu_index is not None and 0 <= self.selected_gpu_index < len(self.gpu_names):
|
| 92 |
+
return self.gpu_names[self.selected_gpu_index]
|
| 93 |
+
return self.gpu_names[0] if self.gpu_names else None
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _format_gib(byte_count: int | float | None) -> str:
|
| 97 |
+
if byte_count is None:
|
| 98 |
+
return "unknown"
|
| 99 |
+
return f"{byte_count / _GIB:.1f} GiB"
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _normalize_model_name(model_name: Optional[str]) -> str:
|
| 103 |
+
if not model_name:
|
| 104 |
+
return ""
|
| 105 |
+
return str(model_name).strip().lower()
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def _canonical_compute(compute_type: Optional[str]) -> str:
|
| 110 |
+
key = (compute_type or "float16").lower()
|
| 111 |
+
return _COMPUTE_CANONICAL.get(key, "float16")
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _estimate_required_vram_bytes(model_name: Optional[str], compute_type: Optional[str]) -> int:
|
| 115 |
+
normalized = _normalize_model_name(model_name)
|
| 116 |
+
canonical = _canonical_compute(compute_type)
|
| 117 |
+
model_table = _MODEL_VRAM_REQUIREMENTS_GB.get(normalized)
|
| 118 |
+
if model_table is not None:
|
| 119 |
+
requirement_gb = model_table.get(canonical)
|
| 120 |
+
if requirement_gb is None:
|
| 121 |
+
requirement_gb = _DEFAULT_MODEL_VRAM_GB.get(canonical, _FALLBACK_MODEL_VRAM_GB)
|
| 122 |
+
else:
|
| 123 |
+
requirement_gb = _DEFAULT_MODEL_VRAM_GB.get(canonical, _FALLBACK_MODEL_VRAM_GB)
|
| 124 |
+
return int(math.ceil(requirement_gb * _GIB))
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _gpu_supports_fp16(capability: Optional[Tuple[int, int]]) -> bool:
|
| 128 |
+
if capability is None:
|
| 129 |
+
return False
|
| 130 |
+
major, minor = capability
|
| 131 |
+
if major is None or minor is None:
|
| 132 |
+
return False
|
| 133 |
+
return (major > 5) or (major == 5 and minor >= 3)
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def _probe_nvidia_smi() -> Optional[DeviceInfo]:
|
| 137 |
+
"""Try to query ``nvidia-smi`` for GPU information."""
|
| 138 |
+
|
| 139 |
+
if shutil.which("nvidia-smi") is None:
|
| 140 |
+
return None
|
| 141 |
+
|
| 142 |
+
try:
|
| 143 |
+
out = subprocess.check_output(
|
| 144 |
+
[
|
| 145 |
+
"nvidia-smi",
|
| 146 |
+
"--query-gpu=name,memory.total,driver_version",
|
| 147 |
+
"--format=csv,noheader,nounits",
|
| 148 |
+
],
|
| 149 |
+
text=True,
|
| 150 |
+
stderr=subprocess.DEVNULL,
|
| 151 |
+
)
|
| 152 |
+
except Exception:
|
| 153 |
+
return None
|
| 154 |
+
|
| 155 |
+
lines = [line.strip() for line in out.splitlines() if line.strip()]
|
| 156 |
+
if not lines:
|
| 157 |
+
return None
|
| 158 |
+
|
| 159 |
+
gpu_names: List[str] = []
|
| 160 |
+
gpu_vram: List[int] = []
|
| 161 |
+
driver_version: Optional[str] = None
|
| 162 |
+
|
| 163 |
+
for line in lines:
|
| 164 |
+
parts = [part.strip() for part in line.split(",")]
|
| 165 |
+
if not parts:
|
| 166 |
+
continue
|
| 167 |
+
gpu_names.append(parts[0])
|
| 168 |
+
if len(parts) > 1:
|
| 169 |
+
try:
|
| 170 |
+
gpu_vram.append(int(float(parts[1])) * 1024 * 1024)
|
| 171 |
+
except (TypeError, ValueError):
|
| 172 |
+
gpu_vram.append(0)
|
| 173 |
+
if len(parts) > 2 and driver_version is None:
|
| 174 |
+
driver_version = parts[2]
|
| 175 |
+
|
| 176 |
+
return DeviceInfo(
|
| 177 |
+
backend="cuda",
|
| 178 |
+
n_gpus=len(gpu_names),
|
| 179 |
+
gpu_names=gpu_names,
|
| 180 |
+
gpu_vram_bytes=gpu_vram,
|
| 181 |
+
driver_version=driver_version,
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def _suppress_known_torch_warnings() -> None:
|
| 186 |
+
"""Silence noisy torch.cuda capability warnings on older GPUs."""
|
| 187 |
+
|
| 188 |
+
global _TORCH_WARNINGS_CONFIGURED
|
| 189 |
+
if _TORCH_WARNINGS_CONFIGURED:
|
| 190 |
+
return
|
| 191 |
+
|
| 192 |
+
patterns = [
|
| 193 |
+
r"torch\.cuda",
|
| 194 |
+
r"torch\._C",
|
| 195 |
+
]
|
| 196 |
+
for module_pattern in patterns:
|
| 197 |
+
warnings.filterwarnings(
|
| 198 |
+
"ignore",
|
| 199 |
+
category=UserWarning,
|
| 200 |
+
module=module_pattern,
|
| 201 |
+
)
|
| 202 |
+
|
| 203 |
+
message_patterns = [
|
| 204 |
+
r"Found GPU\d+ .*cuda capability",
|
| 205 |
+
r"Please install PyTorch with a following CUDA",
|
| 206 |
+
r"not compatible with the current PyTorch installation",
|
| 207 |
+
]
|
| 208 |
+
for message_pattern in message_patterns:
|
| 209 |
+
warnings.filterwarnings(
|
| 210 |
+
"ignore",
|
| 211 |
+
category=UserWarning,
|
| 212 |
+
message=message_pattern,
|
| 213 |
+
)
|
| 214 |
+
|
| 215 |
+
_TORCH_WARNINGS_CONFIGURED = True
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
@contextmanager
|
| 219 |
+
def _suppress_torch_cuda_calls():
|
| 220 |
+
with warnings.catch_warnings():
|
| 221 |
+
warnings.filterwarnings(
|
| 222 |
+
"ignore",
|
| 223 |
+
category=UserWarning,
|
| 224 |
+
module=r"torch\.cuda",
|
| 225 |
+
)
|
| 226 |
+
warnings.filterwarnings(
|
| 227 |
+
"ignore",
|
| 228 |
+
category=UserWarning,
|
| 229 |
+
module=r"torch\._C",
|
| 230 |
+
)
|
| 231 |
+
yield
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def _load_torch_module():
|
| 235 |
+
existing = sys.modules.get("torch")
|
| 236 |
+
if existing is not None:
|
| 237 |
+
return existing
|
| 238 |
+
|
| 239 |
+
with warnings.catch_warnings(record=True) as caught:
|
| 240 |
+
warnings.simplefilter("always")
|
| 241 |
+
module = importlib.import_module("torch")
|
| 242 |
+
|
| 243 |
+
for warning_msg in caught:
|
| 244 |
+
filename = getattr(warning_msg, "filename", "") or ""
|
| 245 |
+
normalized = filename.replace("\\", "/")
|
| 246 |
+
if isinstance(warning_msg.message, UserWarning) and "torch/cuda" in normalized:
|
| 247 |
+
continue
|
| 248 |
+
warnings.showwarning(
|
| 249 |
+
warning_msg.message,
|
| 250 |
+
warning_msg.category,
|
| 251 |
+
warning_msg.filename,
|
| 252 |
+
warning_msg.lineno,
|
| 253 |
+
)
|
| 254 |
+
|
| 255 |
+
return module
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
def _resolve_compute_type(
|
| 259 |
+
backend: str,
|
| 260 |
+
requested: Optional[str],
|
| 261 |
+
supports_fp16: bool,
|
| 262 |
+
auto_mode: bool,
|
| 263 |
+
) -> Tuple[str, List[str]]:
|
| 264 |
+
req = (requested or "auto").lower()
|
| 265 |
+
issues: List[str] = []
|
| 266 |
+
|
| 267 |
+
if req in {"auto", "default"}:
|
| 268 |
+
if backend == "cuda":
|
| 269 |
+
if not supports_fp16:
|
| 270 |
+
issues.append("FP16 may be unsupported on this GPU; it could run slower or fail. Consider int8_float16 if issues occur.")
|
| 271 |
+
return "float16", issues
|
| 272 |
+
if backend == "mps":
|
| 273 |
+
return "float16", issues
|
| 274 |
+
return "float32", issues
|
| 275 |
+
|
| 276 |
+
if backend == "cpu" and req in _GPU_ONLY_COMPUTE_TYPES:
|
| 277 |
+
replacement = "float32" if auto_mode else req
|
| 278 |
+
issues.append(f"{req} requires a GPU; using {replacement}.")
|
| 279 |
+
return replacement, issues
|
| 280 |
+
|
| 281 |
+
if backend in {"cuda", "mps"} and req in _FP16_COMPUTE_TYPES and not supports_fp16:
|
| 282 |
+
issues.append(f"{req} may be unsupported on detected GPU; it could run slower or fail. Consider int8_float16 or float32 if problems occur.")
|
| 283 |
+
return req, issues
|
| 284 |
+
|
| 285 |
+
return req, issues
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
def select_torch_device(
|
| 289 |
+
preferred: str = "auto",
|
| 290 |
+
*,
|
| 291 |
+
model_name: Optional[str] = None,
|
| 292 |
+
compute_type: Optional[str] = None,
|
| 293 |
+
) -> DeviceInfo:
|
| 294 |
+
"""Select and validate the best compute device."""
|
| 295 |
+
|
| 296 |
+
pref = (preferred or "auto").lower()
|
| 297 |
+
auto_mode = pref == "auto"
|
| 298 |
+
requested_compute = (compute_type or "auto").lower()
|
| 299 |
+
|
| 300 |
+
info = DeviceInfo(backend="cpu")
|
| 301 |
+
|
| 302 |
+
torch_module = None
|
| 303 |
+
torch_import_error = None
|
| 304 |
+
try:
|
| 305 |
+
_suppress_known_torch_warnings()
|
| 306 |
+
torch_module = _load_torch_module()
|
| 307 |
+
except Exception as exc: # pragma: no cover - depends on environment
|
| 308 |
+
torch_import_error = exc
|
| 309 |
+
|
| 310 |
+
cuda_available = False
|
| 311 |
+
cuda_device_count = 0
|
| 312 |
+
cuda_names: List[str] = []
|
| 313 |
+
cuda_vram: List[int] = []
|
| 314 |
+
cuda_capabilities: List[Tuple[int, int]] = []
|
| 315 |
+
|
| 316 |
+
if torch_module is not None and getattr(torch_module, "cuda", None) is not None:
|
| 317 |
+
try:
|
| 318 |
+
with _suppress_torch_cuda_calls():
|
| 319 |
+
cuda_available = bool(torch_module.cuda.is_available())
|
| 320 |
+
except Exception:
|
| 321 |
+
cuda_available = False
|
| 322 |
+
|
| 323 |
+
try:
|
| 324 |
+
with _suppress_torch_cuda_calls():
|
| 325 |
+
cuda_device_count = int(torch_module.cuda.device_count())
|
| 326 |
+
except Exception:
|
| 327 |
+
cuda_device_count = 0
|
| 328 |
+
|
| 329 |
+
if cuda_available and cuda_device_count:
|
| 330 |
+
for idx in range(cuda_device_count):
|
| 331 |
+
name = None
|
| 332 |
+
total_mem = None
|
| 333 |
+
capability = None
|
| 334 |
+
try:
|
| 335 |
+
with _suppress_torch_cuda_calls():
|
| 336 |
+
props = torch_module.cuda.get_device_properties(idx)
|
| 337 |
+
except Exception:
|
| 338 |
+
props = None
|
| 339 |
+
|
| 340 |
+
if props is not None:
|
| 341 |
+
name = getattr(props, "name", None)
|
| 342 |
+
total_mem = getattr(props, "total_memory", None)
|
| 343 |
+
capability = (
|
| 344 |
+
getattr(props, "major", None),
|
| 345 |
+
getattr(props, "minor", None),
|
| 346 |
+
)
|
| 347 |
+
|
| 348 |
+
if name is None:
|
| 349 |
+
try:
|
| 350 |
+
name = torch_module.cuda.get_device_name(idx)
|
| 351 |
+
except Exception:
|
| 352 |
+
name = f"CUDA GPU {idx}"
|
| 353 |
+
|
| 354 |
+
cuda_names.append(str(name))
|
| 355 |
+
cuda_vram.append(int(total_mem) if total_mem is not None else 0)
|
| 356 |
+
if capability is not None and capability[0] is not None and capability[1] is not None:
|
| 357 |
+
cuda_capabilities.append((int(capability[0]), int(capability[1])))
|
| 358 |
+
else:
|
| 359 |
+
cuda_capabilities.append((0, 0))
|
| 360 |
+
|
| 361 |
+
mps_available = False
|
| 362 |
+
if torch_module is not None:
|
| 363 |
+
mps_backend = getattr(getattr(torch_module, "backends", None), "mps", None)
|
| 364 |
+
if mps_backend is not None and hasattr(mps_backend, "is_available"):
|
| 365 |
+
try:
|
| 366 |
+
mps_available = bool(mps_backend.is_available())
|
| 367 |
+
except Exception:
|
| 368 |
+
mps_available = False
|
| 369 |
+
|
| 370 |
+
smi_info = _probe_nvidia_smi()
|
| 371 |
+
if not cuda_names and smi_info is not None:
|
| 372 |
+
cuda_names = smi_info.gpu_names
|
| 373 |
+
cuda_vram = smi_info.gpu_vram_bytes
|
| 374 |
+
info.driver_version = smi_info.driver_version
|
| 375 |
+
info.n_gpus = smi_info.n_gpus
|
| 376 |
+
|
| 377 |
+
if torch_module is not None:
|
| 378 |
+
info.cuda_version = getattr(getattr(torch_module, "version", None), "cuda", None)
|
| 379 |
+
|
| 380 |
+
if cuda_available and cuda_device_count:
|
| 381 |
+
info.backend = "cuda"
|
| 382 |
+
info.n_gpus = cuda_device_count
|
| 383 |
+
info.gpu_names = cuda_names
|
| 384 |
+
info.gpu_vram_bytes = cuda_vram
|
| 385 |
+
info.gpu_capabilities = cuda_capabilities
|
| 386 |
+
info.selected_gpu_index = 0 if cuda_device_count else None
|
| 387 |
+
elif pref == "cuda":
|
| 388 |
+
info.backend = "cpu"
|
| 389 |
+
info.issues.append("CUDA backend unavailable in PyTorch; using CPU.")
|
| 390 |
+
if torch_import_error is not None:
|
| 391 |
+
info.notes.append(f"PyTorch import failed: {torch_import_error}")
|
| 392 |
+
elif pref == "mps":
|
| 393 |
+
if mps_available:
|
| 394 |
+
info.backend = "mps"
|
| 395 |
+
else:
|
| 396 |
+
info.backend = "cpu"
|
| 397 |
+
info.issues.append("MPS backend unavailable; using CPU.")
|
| 398 |
+
elif pref == "rocm":
|
| 399 |
+
info.backend = "cpu"
|
| 400 |
+
info.issues.append("ROCm backend not implemented; using CPU.")
|
| 401 |
+
elif pref == "cpu":
|
| 402 |
+
info.backend = "cpu"
|
| 403 |
+
else: # auto mode
|
| 404 |
+
if cuda_available and cuda_device_count:
|
| 405 |
+
info.backend = "cuda"
|
| 406 |
+
info.n_gpus = cuda_device_count
|
| 407 |
+
info.gpu_names = cuda_names
|
| 408 |
+
info.gpu_vram_bytes = cuda_vram
|
| 409 |
+
info.gpu_capabilities = cuda_capabilities
|
| 410 |
+
info.selected_gpu_index = 0 if cuda_device_count else None
|
| 411 |
+
elif mps_available:
|
| 412 |
+
info.backend = "mps"
|
| 413 |
+
else:
|
| 414 |
+
info.backend = "cpu"
|
| 415 |
+
|
| 416 |
+
primary_gpu = info.primary_gpu_name()
|
| 417 |
+
if primary_gpu is not None:
|
| 418 |
+
info.messages.append(f"Detected {primary_gpu} GPU")
|
| 419 |
+
|
| 420 |
+
if info.backend == "cuda" and primary_gpu is None and cuda_names:
|
| 421 |
+
info.messages.append(f"Detected {cuda_names[0]} GPU")
|
| 422 |
+
|
| 423 |
+
initial_backend_for_compute = info.backend
|
| 424 |
+
|
| 425 |
+
available_vram = None
|
| 426 |
+
if initial_backend_for_compute == "cuda" and info.selected_gpu_index is not None:
|
| 427 |
+
if 0 <= info.selected_gpu_index < len(info.gpu_vram_bytes):
|
| 428 |
+
available_vram = info.gpu_vram_bytes[info.selected_gpu_index]
|
| 429 |
+
if available_vram is None and initial_backend_for_compute == "cuda" and info.gpu_vram_bytes:
|
| 430 |
+
available_vram = info.gpu_vram_bytes[0]
|
| 431 |
+
|
| 432 |
+
supports_fp16 = False
|
| 433 |
+
if initial_backend_for_compute == "cuda" and info.selected_gpu_index is not None:
|
| 434 |
+
idx = info.selected_gpu_index
|
| 435 |
+
capability = None
|
| 436 |
+
if 0 <= idx < len(info.gpu_capabilities):
|
| 437 |
+
capability = info.gpu_capabilities[idx]
|
| 438 |
+
supports_fp16 = _gpu_supports_fp16(capability)
|
| 439 |
+
elif initial_backend_for_compute == "mps":
|
| 440 |
+
supports_fp16 = True
|
| 441 |
+
|
| 442 |
+
resolved_compute_candidate, compute_issues = _resolve_compute_type(
|
| 443 |
+
initial_backend_for_compute,
|
| 444 |
+
requested_compute,
|
| 445 |
+
supports_fp16,
|
| 446 |
+
auto_mode,
|
| 447 |
+
)
|
| 448 |
+
|
| 449 |
+
requirement_bytes = None
|
| 450 |
+
compute_label = _canonical_compute(resolved_compute_candidate)
|
| 451 |
+
if initial_backend_for_compute == "cuda":
|
| 452 |
+
requirement_bytes = _estimate_required_vram_bytes(model_name, resolved_compute_candidate)
|
| 453 |
+
|
| 454 |
+
if (
|
| 455 |
+
initial_backend_for_compute == "cuda"
|
| 456 |
+
and available_vram is not None
|
| 457 |
+
and requirement_bytes is not None
|
| 458 |
+
and available_vram < requirement_bytes
|
| 459 |
+
):
|
| 460 |
+
message = (
|
| 461 |
+
f"VRAM too low for model {model_name or 'selected model'} using compute type {compute_label} (~{_format_gib(requirement_bytes)} required, found {_format_gib(available_vram)})."
|
| 462 |
+
)
|
| 463 |
+
if auto_mode:
|
| 464 |
+
message += " Falling back to CPU."
|
| 465 |
+
else:
|
| 466 |
+
message += " GPU execution may fail."
|
| 467 |
+
message += " Consider lowering the compute type (e.g. --transcription_compute_type=int8_float16) or selecting a smaller model via --transcription_model."
|
| 468 |
+
if auto_mode:
|
| 469 |
+
message += " To force GPU usage, rerun with --transcription_device=cuda."
|
| 470 |
+
info.issues.append(message)
|
| 471 |
+
if auto_mode:
|
| 472 |
+
info.backend = "cpu"
|
| 473 |
+
info.selected_gpu_index = None
|
| 474 |
+
|
| 475 |
+
if info.backend != "cuda" and auto_mode and not primary_gpu and smi_info is not None and smi_info.gpu_names:
|
| 476 |
+
info.messages.append("No compatible GPU ready; using CPU.")
|
| 477 |
+
|
| 478 |
+
if info.backend in {"cuda", "mps"}:
|
| 479 |
+
info.resolved_compute_type = resolved_compute_candidate
|
| 480 |
+
info.issues.extend(compute_issues)
|
| 481 |
+
else:
|
| 482 |
+
info.resolved_compute_type = None
|
| 483 |
+
|
| 484 |
+
if info.backend == "cpu" and primary_gpu is None and not info.messages:
|
| 485 |
+
info.messages.append("Using CPU for transcription.")
|
| 486 |
+
|
| 487 |
+
if (
|
| 488 |
+
info.backend == "cuda"
|
| 489 |
+
and not supports_fp16
|
| 490 |
+
and info.resolved_compute_type == "float32"
|
| 491 |
+
):
|
| 492 |
+
info.notes.append("Consider reinstalling PyTorch with newer CUDA support for FP16 acceleration.")
|
| 493 |
+
|
| 494 |
+
return info
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
def select_torch_device_str(
|
| 498 |
+
preferred: str = "auto",
|
| 499 |
+
*,
|
| 500 |
+
model_name: Optional[str] = None,
|
| 501 |
+
compute_type: Optional[str] = None,
|
| 502 |
+
) -> str:
|
| 503 |
+
"""Compatibility helper returning just the backend string."""
|
| 504 |
+
|
| 505 |
+
info = select_torch_device(preferred=preferred, model_name=model_name, compute_type=compute_type)
|
| 506 |
+
return info.backend
|
docker-compose.yml
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
services:
|
| 2 |
+
legen:
|
| 3 |
+
build:
|
| 4 |
+
context: .
|
| 5 |
+
args:
|
| 6 |
+
PYTORCH_INSTALL_CUDA: ${PYTORCH_INSTALL_CUDA:-true}
|
| 7 |
+
PYTORCH_CUDA_INDEX_URL: ${PYTORCH_CUDA_INDEX_URL:-https://download.pytorch.org/whl/cu121}
|
| 8 |
+
image: legen:local
|
| 9 |
+
volumes:
|
| 10 |
+
- ./data:/data
|
| 11 |
+
- ./downloads:/app/downloads
|
| 12 |
+
- ./softsubs_m:/app/softsubs_m
|
| 13 |
+
- ./hardsubs_m:/app/hardsubs_m
|
| 14 |
+
environment:
|
| 15 |
+
NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}
|
| 16 |
+
NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,video,utility}
|
| 17 |
+
command: ["--help"]
|
download_utils.py
ADDED
|
@@ -0,0 +1,407 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import os
|
| 3 |
+
import re
|
| 4 |
+
import shutil
|
| 5 |
+
import subprocess
|
| 6 |
+
import sys
|
| 7 |
+
import tempfile
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import Iterable, List
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def _resolve_downloader() -> str:
|
| 13 |
+
"""Return the executable name for yt-dlp."""
|
| 14 |
+
# First check if yt-dlp is in the same directory as the python executable
|
| 15 |
+
# This handles cases where legen is installed via 'uv tool install'
|
| 16 |
+
local_downloader = Path(sys.executable).parent / "yt-dlp"
|
| 17 |
+
if local_downloader.exists() and os.access(local_downloader, os.X_OK):
|
| 18 |
+
return str(local_downloader)
|
| 19 |
+
|
| 20 |
+
downloader = "yt-dlp"
|
| 21 |
+
if shutil.which(downloader):
|
| 22 |
+
return downloader
|
| 23 |
+
raise FileNotFoundError(
|
| 24 |
+
"yt-dlp executable not found. Install yt-dlp before using URL downloads."
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _append_downloaded_suffix_to_subtitles(media_path: Path) -> None:
|
| 29 |
+
"""Ensure existing subtitle streams are labeled with a [downloaded] suffix."""
|
| 30 |
+
ffprobe_cmd = [
|
| 31 |
+
"ffprobe",
|
| 32 |
+
"-loglevel",
|
| 33 |
+
"error",
|
| 34 |
+
"-select_streams",
|
| 35 |
+
"s",
|
| 36 |
+
"-show_entries",
|
| 37 |
+
"stream=index:stream_tags=language,title",
|
| 38 |
+
"-of",
|
| 39 |
+
"json",
|
| 40 |
+
"file:" + media_path.expanduser().resolve().as_posix(),
|
| 41 |
+
]
|
| 42 |
+
|
| 43 |
+
try:
|
| 44 |
+
probe = subprocess.run(
|
| 45 |
+
ffprobe_cmd,
|
| 46 |
+
check=True,
|
| 47 |
+
capture_output=True,
|
| 48 |
+
text=True,
|
| 49 |
+
)
|
| 50 |
+
except subprocess.CalledProcessError as exc:
|
| 51 |
+
raise RuntimeError(f"ffprobe failed while inspecting subtitles: {exc}") from exc
|
| 52 |
+
|
| 53 |
+
try:
|
| 54 |
+
data = json.loads(probe.stdout or "{}")
|
| 55 |
+
except json.JSONDecodeError as exc:
|
| 56 |
+
raise RuntimeError("ffprobe returned invalid JSON for subtitle metadata") from exc
|
| 57 |
+
|
| 58 |
+
streams = data.get("streams") or []
|
| 59 |
+
if not streams:
|
| 60 |
+
return
|
| 61 |
+
|
| 62 |
+
metadata_args: List[str] = []
|
| 63 |
+
update_needed = False
|
| 64 |
+
|
| 65 |
+
for subtitle_index, stream in enumerate(streams):
|
| 66 |
+
tags = stream.get("tags") or {}
|
| 67 |
+
title = (tags.get("title") or "").strip()
|
| 68 |
+
language = (tags.get("language") or "").strip()
|
| 69 |
+
|
| 70 |
+
base_title = title or language or f"Subtitle {subtitle_index + 1}"
|
| 71 |
+
if not base_title:
|
| 72 |
+
base_title = f"Subtitle {subtitle_index + 1}"
|
| 73 |
+
|
| 74 |
+
final_title = (
|
| 75 |
+
base_title
|
| 76 |
+
if base_title.endswith(" [downloaded]")
|
| 77 |
+
else f"{base_title} [downloaded]"
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
if title != final_title:
|
| 81 |
+
update_needed = True
|
| 82 |
+
|
| 83 |
+
metadata_args.extend([
|
| 84 |
+
f"-metadata:s:s:{subtitle_index}",
|
| 85 |
+
f"title={final_title}",
|
| 86 |
+
])
|
| 87 |
+
|
| 88 |
+
if not update_needed:
|
| 89 |
+
return
|
| 90 |
+
|
| 91 |
+
with tempfile.NamedTemporaryFile(
|
| 92 |
+
dir=media_path.parent, suffix=media_path.suffix, delete=False
|
| 93 |
+
) as tmp:
|
| 94 |
+
temp_output = Path(tmp.name)
|
| 95 |
+
|
| 96 |
+
ffmpeg_cmd = [
|
| 97 |
+
"ffmpeg",
|
| 98 |
+
"-y",
|
| 99 |
+
"-i",
|
| 100 |
+
"file:" + media_path.as_posix(),
|
| 101 |
+
"-map",
|
| 102 |
+
"0",
|
| 103 |
+
"-c",
|
| 104 |
+
"copy",
|
| 105 |
+
]
|
| 106 |
+
|
| 107 |
+
ffmpeg_cmd.extend(metadata_args)
|
| 108 |
+
ffmpeg_cmd.extend(["-movflags", "+faststart", "file:" + temp_output.as_posix()])
|
| 109 |
+
|
| 110 |
+
try:
|
| 111 |
+
subprocess.run(ffmpeg_cmd, check=True)
|
| 112 |
+
except subprocess.CalledProcessError as exc:
|
| 113 |
+
temp_output.unlink(missing_ok=True)
|
| 114 |
+
raise RuntimeError(f"ffmpeg failed while tagging subtitles: {exc}") from exc
|
| 115 |
+
|
| 116 |
+
os.replace(temp_output, media_path)
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
_FORMAT_SUFFIX_RE = re.compile(r"\.f\d+(?:[a-z0-9_-]+)?$", re.IGNORECASE)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _title_from_destination(destination: str) -> str:
|
| 123 |
+
"""Derive a human-friendly title from a yt-dlp destination path."""
|
| 124 |
+
sanitized = destination.strip().strip('"').strip("'")
|
| 125 |
+
name = Path(sanitized).name
|
| 126 |
+
base, _ = os.path.splitext(name)
|
| 127 |
+
cleaned = _FORMAT_SUFFIX_RE.sub("", base).strip()
|
| 128 |
+
return cleaned or name
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _subtitle_label_from_path(path_str: str) -> str:
|
| 132 |
+
"""Extract a readable subtitle label (language or filename)."""
|
| 133 |
+
sanitized = path_str.strip().strip('"')
|
| 134 |
+
if sanitized.startswith("file:"):
|
| 135 |
+
sanitized = sanitized[5:]
|
| 136 |
+
candidate = Path(sanitized)
|
| 137 |
+
suffixes = candidate.suffixes
|
| 138 |
+
if len(suffixes) >= 2:
|
| 139 |
+
lang_suffix = suffixes[-2]
|
| 140 |
+
if lang_suffix.startswith("."):
|
| 141 |
+
lang = lang_suffix[1:]
|
| 142 |
+
else:
|
| 143 |
+
lang = lang_suffix
|
| 144 |
+
lang = lang.strip()
|
| 145 |
+
if lang:
|
| 146 |
+
return lang
|
| 147 |
+
return candidate.name
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def download_urls(
|
| 151 |
+
urls: Iterable[str],
|
| 152 |
+
output_dir: Path,
|
| 153 |
+
overwrite: bool = False,
|
| 154 |
+
download_remote_subs: bool = False,
|
| 155 |
+
) -> List[Path]:
|
| 156 |
+
"""Download one or more media URLs into output_dir using yt-dlp."""
|
| 157 |
+
downloader = _resolve_downloader()
|
| 158 |
+
output_dir = Path(output_dir).expanduser().resolve()
|
| 159 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 160 |
+
|
| 161 |
+
format_selector = "bestvideo[ext=mp4]+bestaudio[ext=m4a]/bestvideo[ext=mp4]+bestaudio/bestvideo+bestaudio/best"
|
| 162 |
+
command = [
|
| 163 |
+
downloader,
|
| 164 |
+
"--no-warnings",
|
| 165 |
+
"--newline",
|
| 166 |
+
"--progress",
|
| 167 |
+
"--format",
|
| 168 |
+
format_selector,
|
| 169 |
+
"--merge-output-format",
|
| 170 |
+
"mp4",
|
| 171 |
+
"-P",
|
| 172 |
+
str(output_dir),
|
| 173 |
+
"--continue",
|
| 174 |
+
]
|
| 175 |
+
if download_remote_subs:
|
| 176 |
+
command.extend([
|
| 177 |
+
"--embed-subs",
|
| 178 |
+
"--sub-langs",
|
| 179 |
+
"all",
|
| 180 |
+
])
|
| 181 |
+
if overwrite:
|
| 182 |
+
command.append("--force-overwrites")
|
| 183 |
+
else:
|
| 184 |
+
command.append("--no-overwrites")
|
| 185 |
+
|
| 186 |
+
url_list = [url.strip() for url in urls if url and url.strip()]
|
| 187 |
+
if not url_list:
|
| 188 |
+
raise ValueError("No URLs provided to download")
|
| 189 |
+
|
| 190 |
+
command.extend(url_list)
|
| 191 |
+
|
| 192 |
+
process = subprocess.Popen(
|
| 193 |
+
command,
|
| 194 |
+
stdout=subprocess.PIPE,
|
| 195 |
+
stderr=subprocess.STDOUT,
|
| 196 |
+
text=True,
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
current_destination: Path | None = None
|
| 200 |
+
current_status: str | None = None
|
| 201 |
+
current_title: str | None = None
|
| 202 |
+
current_url: str | None = None
|
| 203 |
+
current_title_printed = False
|
| 204 |
+
current_stream_index = 0
|
| 205 |
+
active_stream_is_primary = False
|
| 206 |
+
progress_active = False
|
| 207 |
+
last_progress_len = 0
|
| 208 |
+
error_lines: List[str] = []
|
| 209 |
+
|
| 210 |
+
def current_label(fallback: str | None = None) -> str:
|
| 211 |
+
if current_title:
|
| 212 |
+
return current_title
|
| 213 |
+
if current_url:
|
| 214 |
+
return current_url
|
| 215 |
+
if current_destination is not None:
|
| 216 |
+
return current_destination.name
|
| 217 |
+
return fallback or "download"
|
| 218 |
+
|
| 219 |
+
def finish_progress() -> None:
|
| 220 |
+
nonlocal progress_active, last_progress_len
|
| 221 |
+
if progress_active:
|
| 222 |
+
sys.stdout.write("\r")
|
| 223 |
+
sys.stdout.write(" " * last_progress_len)
|
| 224 |
+
sys.stdout.write("\r")
|
| 225 |
+
sys.stdout.flush()
|
| 226 |
+
last_progress_len = 0
|
| 227 |
+
progress_active = False
|
| 228 |
+
|
| 229 |
+
def update_progress(text: str) -> None:
|
| 230 |
+
nonlocal progress_active, last_progress_len
|
| 231 |
+
try:
|
| 232 |
+
terminal_width = os.get_terminal_size().columns
|
| 233 |
+
except OSError:
|
| 234 |
+
terminal_width = 120
|
| 235 |
+
|
| 236 |
+
if terminal_width > 0 and len(text) > terminal_width:
|
| 237 |
+
chunks = [text[i:i + terminal_width] for i in range(0, len(text), terminal_width)]
|
| 238 |
+
finish_progress()
|
| 239 |
+
for chunk in chunks[:-1]:
|
| 240 |
+
print(chunk, flush=True)
|
| 241 |
+
text = chunks[-1]
|
| 242 |
+
|
| 243 |
+
display_text = text
|
| 244 |
+
if len(display_text) < last_progress_len:
|
| 245 |
+
display_text += " " * (last_progress_len - len(display_text))
|
| 246 |
+
|
| 247 |
+
sys.stdout.write("\r" + display_text)
|
| 248 |
+
sys.stdout.flush()
|
| 249 |
+
last_progress_len = len(display_text)
|
| 250 |
+
progress_active = True
|
| 251 |
+
|
| 252 |
+
try:
|
| 253 |
+
assert process.stdout is not None
|
| 254 |
+
for raw_line in process.stdout:
|
| 255 |
+
if raw_line is None:
|
| 256 |
+
continue
|
| 257 |
+
line = raw_line.rstrip("\n")
|
| 258 |
+
stripped = line.strip()
|
| 259 |
+
if not stripped:
|
| 260 |
+
continue
|
| 261 |
+
|
| 262 |
+
if "Extracting URL:" in stripped:
|
| 263 |
+
url = stripped.split("Extracting URL:", 1)[1].strip()
|
| 264 |
+
if url:
|
| 265 |
+
current_url = url
|
| 266 |
+
current_title = None
|
| 267 |
+
current_destination = None
|
| 268 |
+
current_status = None
|
| 269 |
+
current_title_printed = False
|
| 270 |
+
current_stream_index = 0
|
| 271 |
+
active_stream_is_primary = False
|
| 272 |
+
update_progress(f"Loading [{url}]")
|
| 273 |
+
continue
|
| 274 |
+
|
| 275 |
+
destination_prefix = "[download] Destination:"
|
| 276 |
+
resume_prefix = "[download] Resuming download"
|
| 277 |
+
already_downloaded_text = "has already been downloaded"
|
| 278 |
+
|
| 279 |
+
if stripped.startswith(destination_prefix):
|
| 280 |
+
destination_value = stripped[len(destination_prefix):].strip()
|
| 281 |
+
target = Path(destination_value)
|
| 282 |
+
if not target.is_absolute():
|
| 283 |
+
target = (output_dir / target).resolve()
|
| 284 |
+
current_destination = target
|
| 285 |
+
candidate_title = _title_from_destination(destination_value)
|
| 286 |
+
if current_title != candidate_title:
|
| 287 |
+
current_title = candidate_title
|
| 288 |
+
current_title_printed = False
|
| 289 |
+
current_stream_index = 0
|
| 290 |
+
|
| 291 |
+
is_primary_stream = current_stream_index == 0
|
| 292 |
+
|
| 293 |
+
finish_progress()
|
| 294 |
+
if is_primary_stream:
|
| 295 |
+
display_name = current_label(target.name)
|
| 296 |
+
if target.exists() and not overwrite:
|
| 297 |
+
print(f"Existing: {display_name}", flush=True)
|
| 298 |
+
current_status = "skip"
|
| 299 |
+
else:
|
| 300 |
+
print(f"Downloading: {display_name}", flush=True)
|
| 301 |
+
current_status = "download"
|
| 302 |
+
current_title_printed = True
|
| 303 |
+
else:
|
| 304 |
+
if current_status is None:
|
| 305 |
+
current_status = "download"
|
| 306 |
+
|
| 307 |
+
current_stream_index += 1
|
| 308 |
+
active_stream_is_primary = is_primary_stream
|
| 309 |
+
continue
|
| 310 |
+
|
| 311 |
+
if stripped.startswith(resume_prefix):
|
| 312 |
+
if active_stream_is_primary and current_status != "continue":
|
| 313 |
+
finish_progress()
|
| 314 |
+
display_name = current_label()
|
| 315 |
+
print(f"Downloading: {display_name}", flush=True)
|
| 316 |
+
current_status = "continue"
|
| 317 |
+
continue
|
| 318 |
+
|
| 319 |
+
if already_downloaded_text in stripped:
|
| 320 |
+
remainder = stripped.split("[download]", 1)[-1].strip()
|
| 321 |
+
name_part = remainder.split(already_downloaded_text, 1)[0].strip().rstrip('.')
|
| 322 |
+
if not name_part and current_destination is not None:
|
| 323 |
+
name_part = current_destination.name
|
| 324 |
+
if name_part:
|
| 325 |
+
title_guess = _title_from_destination(name_part)
|
| 326 |
+
if title_guess:
|
| 327 |
+
current_title = title_guess
|
| 328 |
+
finish_progress()
|
| 329 |
+
display_name = current_label(name_part or None)
|
| 330 |
+
if not current_title_printed:
|
| 331 |
+
print(f"Existing: {display_name}", flush=True)
|
| 332 |
+
current_title_printed = True
|
| 333 |
+
current_status = "skip"
|
| 334 |
+
active_stream_is_primary = False
|
| 335 |
+
continue
|
| 336 |
+
|
| 337 |
+
if download_remote_subs:
|
| 338 |
+
if "Writing video subtitles to:" in stripped or "Writing subtitles to:" in stripped:
|
| 339 |
+
_, target_part = stripped.split("to:", 1)
|
| 340 |
+
subtitle_target = target_part.strip()
|
| 341 |
+
finish_progress()
|
| 342 |
+
label = _subtitle_label_from_path(subtitle_target)
|
| 343 |
+
print(f"Downloading subtitle track: {label}", flush=True)
|
| 344 |
+
continue
|
| 345 |
+
if stripped.lower().startswith("[download] downloading subtitle"):
|
| 346 |
+
finish_progress()
|
| 347 |
+
print(stripped, flush=True)
|
| 348 |
+
continue
|
| 349 |
+
|
| 350 |
+
if stripped.startswith("[download]") and "Downloading webpage" in stripped:
|
| 351 |
+
finish_progress()
|
| 352 |
+
current_title = None
|
| 353 |
+
current_title_printed = False
|
| 354 |
+
current_stream_index = 0
|
| 355 |
+
active_stream_is_primary = False
|
| 356 |
+
current_status = None
|
| 357 |
+
continue
|
| 358 |
+
|
| 359 |
+
if stripped.startswith("[Merger] Merging formats into"):
|
| 360 |
+
finish_progress()
|
| 361 |
+
current_title = None
|
| 362 |
+
current_title_printed = False
|
| 363 |
+
current_stream_index = 0
|
| 364 |
+
active_stream_is_primary = False
|
| 365 |
+
current_status = None
|
| 366 |
+
continue
|
| 367 |
+
|
| 368 |
+
if stripped.startswith("[download]"):
|
| 369 |
+
if "%" in stripped:
|
| 370 |
+
update_progress(stripped)
|
| 371 |
+
if "100%" in stripped:
|
| 372 |
+
finish_progress()
|
| 373 |
+
continue
|
| 374 |
+
|
| 375 |
+
if "ERROR" in stripped.upper() or stripped.lower().startswith("error:") or stripped.startswith("[error]"):
|
| 376 |
+
finish_progress()
|
| 377 |
+
print(stripped, flush=True)
|
| 378 |
+
error_lines.append(stripped)
|
| 379 |
+
|
| 380 |
+
return_code = process.wait()
|
| 381 |
+
finish_progress()
|
| 382 |
+
except Exception:
|
| 383 |
+
process.kill()
|
| 384 |
+
finish_progress()
|
| 385 |
+
raise
|
| 386 |
+
|
| 387 |
+
if return_code != 0:
|
| 388 |
+
if not error_lines:
|
| 389 |
+
print(f"{downloader} reported an error. Review the output above for details.", flush=True)
|
| 390 |
+
raise RuntimeError(
|
| 391 |
+
f"{downloader} failed with exit code {return_code}. Check the terminal output for details."
|
| 392 |
+
)
|
| 393 |
+
|
| 394 |
+
downloaded_files = sorted(output_dir.rglob("*.mp4"))
|
| 395 |
+
if not downloaded_files:
|
| 396 |
+
raise RuntimeError(
|
| 397 |
+
f"{downloader} completed without producing MP4 files. Check the URL or provide --overwrite to re-download."
|
| 398 |
+
)
|
| 399 |
+
|
| 400 |
+
if download_remote_subs:
|
| 401 |
+
for media_path in downloaded_files:
|
| 402 |
+
try:
|
| 403 |
+
_append_downloaded_suffix_to_subtitles(media_path)
|
| 404 |
+
except RuntimeError as exc:
|
| 405 |
+
print(f"Warning: unable to tag downloaded subtitles in {media_path.name}: {exc}")
|
| 406 |
+
|
| 407 |
+
return downloaded_files
|
ffmpeg_utils.py
CHANGED
|
@@ -1,165 +1,169 @@
|
|
| 1 |
-
import base64
|
| 2 |
-
import os
|
| 3 |
-
import re
|
| 4 |
-
import subprocess
|
| 5 |
-
from pathlib import Path
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
from
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
cmd_ffmpeg_input_map.extend(["-map", f"{
|
| 27 |
-
"-ignore_unknown"])
|
| 28 |
-
|
| 29 |
-
# detect if input has video channels
|
| 30 |
-
result: str = subprocess.run(["ffprobe", "-i", "file:" + input_media_path.as_posix(), "-show_streams",
|
| 31 |
-
"-select_streams", "V", "-loglevel", "error"], capture_output=True, text=True).stdout
|
| 32 |
-
no_video = True if result is None or "DISPOSITION:attached_pic=0" not in result else False
|
| 33 |
-
|
| 34 |
-
# if input has no video channels, map a 1280x720 black screen
|
| 35 |
-
if no_video:
|
| 36 |
-
# create a background image if media is only audio
|
| 37 |
-
background_tempfile: file_utils.TempFile = file_utils.TempFile(
|
| 38 |
-
"", ".jpeg")
|
| 39 |
-
|
| 40 |
-
# create a 1280x720 jpeg image
|
| 41 |
-
with open(file=background_tempfile.getpath(), mode="wb") as file:
|
| 42 |
-
file.write(base64.b64decode("/9j/4AAQSkZJRgABAQEASABIAAD/4gKwSUNDX1BST0ZJTEUAAQEAAAKgbGNtcwRAAABtbnRyUkdCIFhZWiAH5wAFABkAFwA2ADhhY3NwQVBQTAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9tYAAQAAAADTLWxjbXMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA1kZXNjAAABIAAAAEBjcHJ0AAABYAAAADZ3dHB0AAABmAAAABRjaGFkAAABrAAAACxyWFlaAAAB2AAAABRiWFlaAAAB7AAAABRnWFlaAAACAAAAABRyVFJDAAACFAAAACBnVFJDAAACFAAAACBiVFJDAAACFAAAACBjaHJtAAACNAAAACRkbW5kAAACWAAAACRkbWRkAAACfAAAACRtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACQAAAAcAEcASQBNAFAAIABiAHUAaQBsAHQALQBpAG4AIABzAFIARwBCbWx1YwAAAAAAAAABAAAADGVuVVMAAAAaAAAAHABQAHUAYgBsAGkAYwAgAEQAbwBtAGEAaQBuAABYWVogAAAAAAAA9tYAAQAAAADTLXNmMzIAAAAAAAEMQgAABd7///MlAAAHkwAA/ZD///uh///9ogAAA9wAAMBuWFlaIAAAAAAAAG+gAAA49QAAA5BYWVogAAAAAAAAJJ8AAA+EAAC2xFhZWiAAAAAAAABilwAAt4cAABjZcGFyYQAAAAAAAwAAAAJmZgAA8qcAAA1ZAAAT0AAACltjaHJtAAAAAAADAAAAAKPXAABUfAAATM0AAJmaAAAmZwAAD1xtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAEcASQBNAFBtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEL/2wBDAAoHBwgHBgoICAgLCgoLDhgQDg0NDh0VFhEYIx8lJCIfIiEmKzcvJik0KSEiMEExNDk7Pj4+JS5ESUM8SDc9Pjv/2wBDAQoLCw4NDhwQEBw7KCIoOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozv/wgARCALQBQADASIAAhEBAxEB/8QAGAABAQEBAQAAAAAAAAAAAAAAAAECAwb/xAAUAQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIQAxAAAAHxgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWdDK05tDLUI2MFI3DOmjm0MtwybM3UMFItMrTLeTeLCNUwojcMrCtQyA2MWdDJTCiNwytMt5IAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC757MXYxrOhINywSbJZTOufQxuUlgJoiwxvGhaM6zRneTWdZNZ1g3LkqjOdZNy057uTRBeezOwlxoWCaxsiUwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACwAAAAAAALAAAAsACwAAFg3cwuQAAAAsACwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP/EACAQAAMBAAICAgMAAAAAAAAAAAABERAgMEFgMXBAkND/2gAIAQEAAQUC/QFPoKE7JzQ+p9MJzhPb7jITITIQgljRCEyY8SGTITIeYQmQg8Q9hNpRkITIT2ZbCY2LX1IYh4hi+HrzyLEPHiHi3yLHw8iH7S8Ysg8nB5MWvFrJjzyLFrxasnF5MXyIfx7NfwL0XheaJj6rwv0hS/0nf//EABQRAQAAAAAAAAAAAAAAAAAAALD/2gAIAQMBAT8BYQ//xAAUEQEAAAAAAAAAAAAAAAAAAACw/9oACAECAQE/AWEP/8QAFBABAAAAAAAAAAAAAAAAAAAA0P/aAAgBAQAGPwJhA//EACMQAAMBAQACAwABBQAAAAAAAAABERAxQVEgIWAwQGFwkND/2gAIAQEAAT8h/wBAVf4BSuX/AEelESX8NOv4VQ4+SV/X67jgTovkSTLX3FiBqCogfoUWQQhAwlSBKHGUQkFSGoIQNQgnGYHAlSMpM4OcSpGJ9zGYGTFRA/QSrzA1P0vWPomY4FwgfZ3Ep5OBWdI73E/uZb9ZHe55DQanGcnOHRaT0yvPwtWPhwNBOrfBznA1fIh8Z9OlpxkfvEfA1OP0yd1qLh2NGdII8C4dXPOcCdIGkhs4cb7nRAlPj9D4N4+PjW8DVIH36E6RPGonUQyJYPn6mUW5XtaK8sKyvblbWUu0XLCsr95X7yv2Imf2MSi3Sv2V+8rK3tF2tr95f+mB/wD/2gAMAwEAAgADAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABIBCADCBACCMBCBACCCBCJBADIDCBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFLHGEGHHMMBABNBJKPAHKFCBHJHOKFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEEAMEMAAAEIAAEAIIMBAEIMAEAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAALD/2gAIAQMBAT8QYQ//xAAUEQEAAAAAAAAAAAAAAAAAAACw/9oACAECAQE/EGEP/8QAJxABAAIBAwQBBQADAAAAAAAAAQARMRAhYUFRcZFgIDBwgbGQodD/2gAIAQEAAT8Q/wAAQxcRGn8ALwJTuzsMx9u1Z+srWRSh9kCWMQ7fs1LupVu7+peBKd2PYxKaflgsHeBRRBVBcAWTBCg2wHVlMgTO8W4jveZjiccrL9IFjegWFy8tnHKyzERUQLmCmusz6BY3mCCrhAqw0Be8Gt92ccv8QqyxPBEVssO1cqWYl/iccpedoNavfT+8wedHe0C5iU22lt6BRRNzY2hVvsygTMzHYnHNm/SFQxHRZxzOMfJcOgRDALKiC1Jj8SxQRWLMRShqJlaG/GF9A8TaLjxo5gq9FsQVemR61oK/aog26xbX0mfTL5mDzMEdFgBYy5dhKdlfaGCOlaGZ/ExxaVGsZgaf3mDzofeJhSBClubE0Q3CAFkzwKKijZXQUotJ7EW1zP8AJjHMQcgzBLVGCY/Ew+JkMMQFOIF0vzKNhvEIqIbGZg1fmCJZAFgLuALIs3ArSEKPWIJSQDYbTPoskQFJCjY6TFcMof7ncXEUHWGCOn+cz+IZu/UQcl6YGn99Sq36g5SkxEdgIY5iy0gAbbEtUYIXIRRcRWk3Rl8TJmb5NicsVk6UdX3FXLehgMW66CwYtllHV96CmGopyrBTDU5Yq5b0CwxS2rFOV15Za7tirluCwZyzM5HuW3d7zke4KYanI9y0tbiq3P2zaItw7aCMKTke5yPegXWOQ68sU5dLrE5Yq5bgphqcj3BTCkU5X3+ChRsgupFOCpn/AKTt/9k="))
|
| 43 |
-
|
| 44 |
-
# get input media duration
|
| 45 |
-
duration_sec = subprocess.run(["ffprobe", "-i", "file:" + input_media_path.as_posix(), "-show_entries", "format=duration", "-v", "quiet", "-of", "csv=p=0"],
|
| 46 |
-
capture_output=True, text=True).stdout.split("\n")[0].split('.')[0]
|
| 47 |
-
|
| 48 |
-
cmd_ffmpeg.extend(["-t", f"{duration_sec}", "-loop", "1",
|
| 49 |
-
"-framerate", "10", "-i", background_tempfile.getpath().as_posix()])
|
| 50 |
-
map_index = cmd_ffmpeg.count("-i") - 1
|
| 51 |
-
# unmap #0, remap #0 with no video, map #1 (generated video from image)
|
| 52 |
-
cmd_ffmpeg_input_map.extend(
|
| 53 |
-
["-map", "-0", "-map", f"
|
| 54 |
-
|
| 55 |
-
# map each subtitle
|
| 56 |
-
for i, subtitle in enumerate(subtitles_path):
|
| 57 |
-
cmd_ffmpeg.extend(["-i", "file:" + subtitle.as_posix()])
|
| 58 |
-
map_index = cmd_ffmpeg.count("-i") - 1
|
| 59 |
-
cmd_ffmpeg_input_map.extend(["-map", f"{map_index}:s"])
|
| 60 |
-
|
| 61 |
-
#
|
| 62 |
-
|
| 63 |
-
(
|
| 64 |
-
|
| 65 |
-
#
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
if
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
#
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
#
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
#
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
#
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
import os
|
| 3 |
+
import re
|
| 4 |
+
import subprocess
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
from ffmpeg_progress_yield import FfmpegProgress
|
| 8 |
+
from tqdm import tqdm
|
| 9 |
+
|
| 10 |
+
import file_utils
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def insert_subtitle(input_media_path: Path, subtitles_path: list[Path], burn_subtitles: bool, output_video_path: Path, codec_video: str = "h264", codec_audio: str = "aac"):
|
| 14 |
+
# use only valid srt files
|
| 15 |
+
subtitles_path: list[Path] = file_utils.validate_files(subtitles_path)
|
| 16 |
+
|
| 17 |
+
# insert in comand the basics of ffmpeg
|
| 18 |
+
cmd_ffmpeg = ["ffmpeg", "-y"]
|
| 19 |
+
cmd_ffmpeg_input_map = []
|
| 20 |
+
|
| 21 |
+
# add ffmpeg input main media
|
| 22 |
+
cmd_ffmpeg.extend(["-i", "file:" + input_media_path.as_posix()])
|
| 23 |
+
# map video stream, audio, subtitle, data and metadata
|
| 24 |
+
source_input_index = cmd_ffmpeg.count("-i") - 1
|
| 25 |
+
existing_subtitle_map = ["-map", f"{source_input_index}:s?"]
|
| 26 |
+
cmd_ffmpeg_input_map.extend(["-map", f"{source_input_index}:V?", "-map", f"{source_input_index}:a",
|
| 27 |
+
"-ignore_unknown"])
|
| 28 |
+
|
| 29 |
+
# detect if input has video channels
|
| 30 |
+
result: str = subprocess.run(["ffprobe", "-i", "file:" + input_media_path.as_posix(), "-show_streams",
|
| 31 |
+
"-select_streams", "V", "-loglevel", "error"], capture_output=True, text=True).stdout
|
| 32 |
+
no_video = True if result is None or "DISPOSITION:attached_pic=0" not in result else False
|
| 33 |
+
|
| 34 |
+
# if input has no video channels, map a 1280x720 black screen
|
| 35 |
+
if no_video:
|
| 36 |
+
# create a background image if media is only audio
|
| 37 |
+
background_tempfile: file_utils.TempFile = file_utils.TempFile(
|
| 38 |
+
"", ".jpeg")
|
| 39 |
+
|
| 40 |
+
# create a 1280x720 jpeg image
|
| 41 |
+
with open(file=background_tempfile.getpath(), mode="wb") as file:
|
| 42 |
+
file.write(base64.b64decode("/9j/4AAQSkZJRgABAQEASABIAAD/4gKwSUNDX1BST0ZJTEUAAQEAAAKgbGNtcwRAAABtbnRyUkdCIFhZWiAH5wAFABkAFwA2ADhhY3NwQVBQTAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9tYAAQAAAADTLWxjbXMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA1kZXNjAAABIAAAAEBjcHJ0AAABYAAAADZ3dHB0AAABmAAAABRjaGFkAAABrAAAACxyWFlaAAAB2AAAABRiWFlaAAAB7AAAABRnWFlaAAACAAAAABRyVFJDAAACFAAAACBnVFJDAAACFAAAACBiVFJDAAACFAAAACBjaHJtAAACNAAAACRkbW5kAAACWAAAACRkbWRkAAACfAAAACRtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACQAAAAcAEcASQBNAFAAIABiAHUAaQBsAHQALQBpAG4AIABzAFIARwBCbWx1YwAAAAAAAAABAAAADGVuVVMAAAAaAAAAHABQAHUAYgBsAGkAYwAgAEQAbwBtAGEAaQBuAABYWVogAAAAAAAA9tYAAQAAAADTLXNmMzIAAAAAAAEMQgAABd7///MlAAAHkwAA/ZD///uh///9ogAAA9wAAMBuWFlaIAAAAAAAAG+gAAA49QAAA5BYWVogAAAAAAAAJJ8AAA+EAAC2xFhZWiAAAAAAAABilwAAt4cAABjZcGFyYQAAAAAAAwAAAAJmZgAA8qcAAA1ZAAAT0AAACltjaHJtAAAAAAADAAAAAKPXAABUfAAATM0AAJmaAAAmZwAAD1xtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAEcASQBNAFBtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEL/2wBDAAoHBwgHBgoICAgLCgoLDhgQDg0NDh0VFhEYIx8lJCIfIiEmKzcvJik0KSEiMEExNDk7Pj4+JS5ESUM8SDc9Pjv/2wBDAQoLCw4NDhwQEBw7KCIoOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozv/wgARCALQBQADASIAAhEBAxEB/8QAGAABAQEBAQAAAAAAAAAAAAAAAAECAwb/xAAUAQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIQAxAAAAHxgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWdDK05tDLUI2MFI3DOmjm0MtwybM3UMFItMrTLeTeLCNUwojcMrCtQyA2MWdDJTCiNwytMt5IAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC757MXYxrOhINywSbJZTOufQxuUlgJoiwxvGhaM6zRneTWdZNZ1g3LkqjOdZNy057uTRBeezOwlxoWCaxsiUwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACwAAAAAAALAAAAsACwAAFg3cwuQAAAAsACwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP/EACAQAAMBAAICAgMAAAAAAAAAAAABERAgMEFgMXBAkND/2gAIAQEAAQUC/QFPoKE7JzQ+p9MJzhPb7jITITIQgljRCEyY8SGTITIeYQmQg8Q9hNpRkITIT2ZbCY2LX1IYh4hi+HrzyLEPHiHi3yLHw8iH7S8Ysg8nB5MWvFrJjzyLFrxasnF5MXyIfx7NfwL0XheaJj6rwv0hS/0nf//EABQRAQAAAAAAAAAAAAAAAAAAALD/2gAIAQMBAT8BYQ//xAAUEQEAAAAAAAAAAAAAAAAAAACw/9oACAECAQE/AWEP/8QAFBABAAAAAAAAAAAAAAAAAAAA0P/aAAgBAQAGPwJhA//EACMQAAMBAQACAwABBQAAAAAAAAABERAxQVEgIWAwQGFwkND/2gAIAQEAAT8h/wBAVf4BSuX/AEelESX8NOv4VQ4+SV/X67jgTovkSTLX3FiBqCogfoUWQQhAwlSBKHGUQkFSGoIQNQgnGYHAlSMpM4OcSpGJ9zGYGTFRA/QSrzA1P0vWPomY4FwgfZ3Ep5OBWdI73E/uZb9ZHe55DQanGcnOHRaT0yvPwtWPhwNBOrfBznA1fIh8Z9OlpxkfvEfA1OP0yd1qLh2NGdII8C4dXPOcCdIGkhs4cb7nRAlPj9D4N4+PjW8DVIH36E6RPGonUQyJYPn6mUW5XtaK8sKyvblbWUu0XLCsr95X7yv2Imf2MSi3Sv2V+8rK3tF2tr95f+mB/wD/2gAMAwEAAgADAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABIBCADCBACCMBCBACCCBCJBADIDCBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFLHGEGHHMMBABNBJKPAHKFCBHJHOKFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEEAMEMAAAEIAAEAIIMBAEIMAEAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAALD/2gAIAQMBAT8QYQ//xAAUEQEAAAAAAAAAAAAAAAAAAACw/9oACAECAQE/EGEP/8QAJxABAAIBAwQBBQADAAAAAAAAAQARMRAhYUFRcZFgIDBwgbGQodD/2gAIAQEAAT8Q/wAAQxcRGn8ALwJTuzsMx9u1Z+srWRSh9kCWMQ7fs1LupVu7+peBKd2PYxKaflgsHeBRRBVBcAWTBCg2wHVlMgTO8W4jveZjiccrL9IFjegWFy8tnHKyzERUQLmCmusz6BY3mCCrhAqw0Be8Gt92ccv8QqyxPBEVssO1cqWYl/iccpedoNavfT+8wedHe0C5iU22lt6BRRNzY2hVvsygTMzHYnHNm/SFQxHRZxzOMfJcOgRDALKiC1Jj8SxQRWLMRShqJlaG/GF9A8TaLjxo5gq9FsQVemR61oK/aog26xbX0mfTL5mDzMEdFgBYy5dhKdlfaGCOlaGZ/ExxaVGsZgaf3mDzofeJhSBClubE0Q3CAFkzwKKijZXQUotJ7EW1zP8AJjHMQcgzBLVGCY/Ew+JkMMQFOIF0vzKNhvEIqIbGZg1fmCJZAFgLuALIs3ArSEKPWIJSQDYbTPoskQFJCjY6TFcMof7ncXEUHWGCOn+cz+IZu/UQcl6YGn99Sq36g5SkxEdgIY5iy0gAbbEtUYIXIRRcRWk3Rl8TJmb5NicsVk6UdX3FXLehgMW66CwYtllHV96CmGopyrBTDU5Yq5b0CwxS2rFOV15Za7tirluCwZyzM5HuW3d7zke4KYanI9y0tbiq3P2zaItw7aCMKTke5yPegXWOQ68sU5dLrE5Yq5bgphqcj3BTCkU5X3+ChRsgupFOCpn/AKTt/9k="))
|
| 43 |
+
|
| 44 |
+
# get input media duration
|
| 45 |
+
duration_sec = subprocess.run(["ffprobe", "-i", "file:" + input_media_path.as_posix(), "-show_entries", "format=duration", "-v", "quiet", "-of", "csv=p=0"],
|
| 46 |
+
capture_output=True, text=True).stdout.split("\n")[0].split('.')[0]
|
| 47 |
+
|
| 48 |
+
cmd_ffmpeg.extend(["-t", f"{duration_sec}", "-loop", "1",
|
| 49 |
+
"-framerate", "10", "-i", background_tempfile.getpath().as_posix()])
|
| 50 |
+
map_index = cmd_ffmpeg.count("-i") - 1
|
| 51 |
+
# unmap #0, remap #0 with no video, map #1 (generated video from image)
|
| 52 |
+
cmd_ffmpeg_input_map.extend(
|
| 53 |
+
["-map", "-0", "-map", f"{source_input_index}:a", "-map", f"{source_input_index}:d?", "-map", f"{source_input_index}:t?", "-map", f"{map_index}:V", "-ignore_unknown"])
|
| 54 |
+
|
| 55 |
+
# map each subtitle
|
| 56 |
+
for i, subtitle in enumerate(subtitles_path):
|
| 57 |
+
cmd_ffmpeg.extend(["-i", "file:" + subtitle.as_posix()])
|
| 58 |
+
map_index = cmd_ffmpeg.count("-i") - 1
|
| 59 |
+
cmd_ffmpeg_input_map.extend(["-map", f"{map_index}:s"])
|
| 60 |
+
|
| 61 |
+
# ensure original subtitle streams remain, placed after newly generated ones
|
| 62 |
+
if existing_subtitle_map:
|
| 63 |
+
cmd_ffmpeg_input_map.extend(existing_subtitle_map)
|
| 64 |
+
|
| 65 |
+
# add comand to burn subtitles if its demanded and has at least one valid subtitle in the array. Burn the first one. Also ensure hwupload if necessary
|
| 66 |
+
vf_hwupload = True if codec_video.endswith(
|
| 67 |
+
("_nvenc", "_amf", "_v4l2m2m", "_qsv", "_vaapi", "_videotoolbox", "_cuvid")) else False
|
| 68 |
+
hw_device = codec_video.split("_")[-1] if vf_hwupload else None
|
| 69 |
+
# set hw_device as cuda if api is nvenc or cuvid
|
| 70 |
+
if hw_device == "nvenc" or hw_device == "cuvid":
|
| 71 |
+
hw_device = "cuda"
|
| 72 |
+
vf_hwupload = False
|
| 73 |
+
|
| 74 |
+
# set hw_device as vaapi if api is v4l2m2m or amf
|
| 75 |
+
if hw_device == "v4l2m2m" or hw_device == "amf":
|
| 76 |
+
hw_device = "vaapi"
|
| 77 |
+
|
| 78 |
+
if burn_subtitles and len(subtitles_path) > 0:
|
| 79 |
+
# create temp file for .srt
|
| 80 |
+
srt_temp = file_utils.TempFile(
|
| 81 |
+
"", file_ext=".srt")
|
| 82 |
+
|
| 83 |
+
file_utils.copy_file_if_different(
|
| 84 |
+
subtitles_path[0], srt_temp.getpath(), True)
|
| 85 |
+
|
| 86 |
+
# align subtitles to botton center if hass video and to center center if only audio with black screen
|
| 87 |
+
sub_align = 10 if no_video else 2
|
| 88 |
+
|
| 89 |
+
# insert scale, subtitles filter and hwupload if required
|
| 90 |
+
# scale at minimul height of 480p. it also will make the dimensions divisible by 2
|
| 91 |
+
cmd_ffmpeg.extend(
|
| 92 |
+
["-vf", f"format=nv12, scale='ceil((max(480,ih)*iw/ih)/2)*2:ceil(max(480,ih)/2)*2', subtitles=\'{add_ffmpeg_escape_chars(srt_temp.temp_file.name)}\':force_style='Alignment={sub_align},Fontname=Jost,PrimaryColour=&H03fcff,Fontsize=18,BackColour=&H80000000,Bold=1,Spacing=0.09,Outline=1,Shadow=0,MarginL=10,MarginR=10'" + (', hwupload' if vf_hwupload else '')])
|
| 93 |
+
else:
|
| 94 |
+
if vf_hwupload:
|
| 95 |
+
cmd_ffmpeg.extend(["-vf", f"format=nv12, hwupload"])
|
| 96 |
+
burn_subtitles = False
|
| 97 |
+
|
| 98 |
+
cmd_ffmpeg.extend(cmd_ffmpeg_input_map)
|
| 99 |
+
|
| 100 |
+
# init a hw_device if hwupload is set on video filters
|
| 101 |
+
if hw_device is not None:
|
| 102 |
+
cmd_ffmpeg.extend(["-init_hw_device", hw_device])
|
| 103 |
+
|
| 104 |
+
# add the remaining parameters and output path
|
| 105 |
+
cmd_ffmpeg.extend(["-c:V", codec_video, "-c:a", codec_audio, "-c:s", "mov_text",
|
| 106 |
+
"-pix_fmt", "yuv420p", "-movflags", "+faststart",
|
| 107 |
+
"-sws_flags", "bicubic+accurate_rnd+full_chroma_int+full_chroma_inp",
|
| 108 |
+
"file:" + output_video_path.as_posix()])
|
| 109 |
+
|
| 110 |
+
# run FFmpeg command with a fancy progress bar
|
| 111 |
+
ff = FfmpegProgress(cmd_ffmpeg)
|
| 112 |
+
with tqdm(total=100, position=0, ascii="░▒█", desc="Inserting subtitles" if not burn_subtitles else "Burning subtitles", unit="%", unit_scale=True, leave=True, bar_format="{desc} [{bar}] {percentage:3.0f}% | {rate_fmt}{postfix} | ETA: {remaining} | ⏱: {elapsed}") as pbar:
|
| 113 |
+
for progress in ff.run_command_with_progress():
|
| 114 |
+
pbar.update(progress - pbar.n)
|
| 115 |
+
|
| 116 |
+
# destroy unecessary file
|
| 117 |
+
if 'srt_temp' in locals():
|
| 118 |
+
srt_temp.destroy()
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def extract_audio_wav(input_media_path: Path, output_path: Path):
|
| 122 |
+
# set the FFMpeg command
|
| 123 |
+
cmd_ffmpeg = ["ffmpeg", "-y", "-i", "file:" + input_media_path.as_posix(),
|
| 124 |
+
"-vn", "-c:a", "pcm_s16le", "-ac", "1", "-ar", "16000", "file:" + output_path.as_posix()]
|
| 125 |
+
|
| 126 |
+
# run FFmpeg command with a fancy progress bar
|
| 127 |
+
ff = FfmpegProgress(cmd_ffmpeg)
|
| 128 |
+
with tqdm(total=100, position=0, ascii="░▒█", desc="Extracting audio", unit="%", unit_scale=True, leave=True, bar_format="{desc} {percentage:3.0f}% | ETA: {remaining} | ⏱: {elapsed}") as pbar:
|
| 129 |
+
for progress in ff.run_command_with_progress():
|
| 130 |
+
pbar.update(progress - pbar.n)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def extract_short_wav(input_media_path: Path, output_path: Path):
|
| 134 |
+
# get input media duration
|
| 135 |
+
duration_sec = subprocess.run(["ffprobe", "-i", "file:" + input_media_path.as_posix(), "-show_entries", "format=duration", "-v", "quiet", "-of", "csv=p=0"],
|
| 136 |
+
capture_output=True, text=True).stdout.split("\n")[0].replace("\n", "").replace(" ", "").replace(" ", "").replace("00:", "").replace(":", "").split(".")[0]
|
| 137 |
+
|
| 138 |
+
if int(duration_sec) > 240:
|
| 139 |
+
start_sec = int(duration_sec) - 120
|
| 140 |
+
end_sec = int(duration_sec) - 60
|
| 141 |
+
elif int(duration_sec) > 120:
|
| 142 |
+
start_sec = int(duration_sec) - 80
|
| 143 |
+
end_sec = int(duration_sec) - 20
|
| 144 |
+
elif int(duration_sec) > 80:
|
| 145 |
+
start_sec = int(duration_sec) - 60
|
| 146 |
+
end_sec = int(duration_sec) - 20
|
| 147 |
+
else:
|
| 148 |
+
start_sec = 0
|
| 149 |
+
end_sec = int(duration_sec)
|
| 150 |
+
|
| 151 |
+
# set the FFMpeg command
|
| 152 |
+
cmd_ffmpeg = ["ffmpeg", "-y", "-ss", f"{start_sec}", "-t", f"{end_sec}", "-i", "file:" + input_media_path.as_posix(),
|
| 153 |
+
"-vn", "-c:a", "pcm_s16le", "-af", "loudnorm", "-ac", "1", "-ar", "16000", "file:" + output_path.as_posix()]
|
| 154 |
+
|
| 155 |
+
# run FFmpeg command with a fancy progress bar
|
| 156 |
+
ff = FfmpegProgress(cmd_ffmpeg)
|
| 157 |
+
with tqdm(total=100, position=0, ascii="░▒█", desc="Extracting audio", unit="%", unit_scale=True, leave=False, bar_format="{desc} {percentage:3.0f}% | ETA: {remaining} | ⏱: {elapsed}") as pbar:
|
| 158 |
+
for progress in ff.run_command_with_progress():
|
| 159 |
+
pbar.update(progress - pbar.n)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def add_ffmpeg_escape_chars(string):
|
| 163 |
+
new_string = ""
|
| 164 |
+
for char in string:
|
| 165 |
+
if os.name == 'nt':
|
| 166 |
+
if char == ":" or char == "\x5c":
|
| 167 |
+
new_string += "\x5c"
|
| 168 |
+
new_string += char
|
| 169 |
+
return new_string
|
file_utils.py
CHANGED
|
@@ -1,170 +1,170 @@
|
|
| 1 |
-
import filecmp
|
| 2 |
-
import os
|
| 3 |
-
import shutil
|
| 4 |
-
import tempfile
|
| 5 |
-
from inspect import currentframe, getframeinfo
|
| 6 |
-
from pathlib import Path
|
| 7 |
-
|
| 8 |
-
# return the itens in array that is not inexisting or empty
|
| 9 |
-
def validate_files(paths):
|
| 10 |
-
valid_files = [path for path in paths if file_is_valid(path)]
|
| 11 |
-
return valid_files
|
| 12 |
-
|
| 13 |
-
# check if a file is existing and not empty
|
| 14 |
-
def file_is_valid(path):
|
| 15 |
-
if path is not None and path.is_file():
|
| 16 |
-
size = path.stat().st_size
|
| 17 |
-
if size > 0:
|
| 18 |
-
return True
|
| 19 |
-
return False
|
| 20 |
-
|
| 21 |
-
# validate if an string is a valid dir or path with valid content
|
| 22 |
-
def check_valid_path(path_str):
|
| 23 |
-
# create a Path object from the input string
|
| 24 |
-
path = Path(path_str)
|
| 25 |
-
|
| 26 |
-
# check if the path exists
|
| 27 |
-
if not path.exists():
|
| 28 |
-
raise FileNotFoundError(f"The path '{path_str}' does not exist.")
|
| 29 |
-
|
| 30 |
-
# check if it's a directory with at least one file or a valid file with content
|
| 31 |
-
if path.is_dir():
|
| 32 |
-
# check if the directory has at least one file with content
|
| 33 |
-
files_with_content = [file for file in path.iterdir() if file.is_file() and file.stat().st_size > 0]
|
| 34 |
-
if not files_with_content:
|
| 35 |
-
raise ValueError(f"The directory '{path_str}' does not contain any files with content.")
|
| 36 |
-
elif path.is_file():
|
| 37 |
-
# check if the file has content
|
| 38 |
-
if path.stat().st_size == 0:
|
| 39 |
-
raise ValueError(f"The file '{path_str}' does not contain any content.")
|
| 40 |
-
else:
|
| 41 |
-
raise ValueError(f"The path '{path_str}' is neither a valid directory nor a valid file.")
|
| 42 |
-
|
| 43 |
-
return path
|
| 44 |
-
|
| 45 |
-
# validate if an string is a valid dir or path with valid content
|
| 46 |
-
def check_existing_path(path_str):
|
| 47 |
-
# create a Path object from the input string
|
| 48 |
-
path = Path(path_str)
|
| 49 |
-
|
| 50 |
-
# check if the path exists
|
| 51 |
-
if not path.exists():
|
| 52 |
-
raise FileNotFoundError(f"The path '{path_str}' does not exist.")
|
| 53 |
-
|
| 54 |
-
# check if it's a directory with at least one file or a valid file with content
|
| 55 |
-
if not path.is_dir() and not path.is_file():
|
| 56 |
-
raise ValueError(f"The path '{path_str}' is neither a valid directory nor a valid file.")
|
| 57 |
-
|
| 58 |
-
return path_str
|
| 59 |
-
|
| 60 |
-
# create a tempfile class to use as object
|
| 61 |
-
class TempFile:
|
| 62 |
-
|
| 63 |
-
def __init__(self, final_path: Path, file_ext: str = None):
|
| 64 |
-
self.final_path: Path = None if final_path is None else Path(
|
| 65 |
-
final_path)
|
| 66 |
-
self.file_ext = file_ext
|
| 67 |
-
os.makedirs(Path(Path(getframeinfo(
|
| 68 |
-
currentframe()).filename).resolve().parent, "temp"), exist_ok=True)
|
| 69 |
-
self.temp_file: tempfile.NamedTemporaryFile = tempfile.NamedTemporaryFile(dir=Path(Path(getframeinfo(currentframe()).filename).resolve().parent, "temp"),
|
| 70 |
-
delete=False, suffix=file_ext)
|
| 71 |
-
|
| 72 |
-
self.temp_file_name = self.temp_file.name
|
| 73 |
-
self.temp_file_path: Path = Path(self.temp_file.name)
|
| 74 |
-
|
| 75 |
-
self.temp_file.close()
|
| 76 |
-
|
| 77 |
-
# return the actual path of the file
|
| 78 |
-
def getpath(self):
|
| 79 |
-
if self.temp_file_path.is_file():
|
| 80 |
-
return self.temp_file_path
|
| 81 |
-
elif file_is_valid(self.final_path):
|
| 82 |
-
return self.final_path
|
| 83 |
-
else:
|
| 84 |
-
return None
|
| 85 |
-
|
| 86 |
-
# return the actual path of the file if not empty
|
| 87 |
-
def getvalidpath(self):
|
| 88 |
-
if file_is_valid(self.temp_file_path):
|
| 89 |
-
return self.temp_file_path
|
| 90 |
-
elif file_is_valid(self.final_path):
|
| 91 |
-
return self.final_path
|
| 92 |
-
else:
|
| 93 |
-
return None
|
| 94 |
-
|
| 95 |
-
# save the temp file into final path
|
| 96 |
-
def save(self, overwrite_if_valid: bool = True, update_path: Path = None):
|
| 97 |
-
# update final path case user specifies it
|
| 98 |
-
if update_path is None:
|
| 99 |
-
path: Path = self.final_path
|
| 100 |
-
else:
|
| 101 |
-
path: Path = update_path
|
| 102 |
-
|
| 103 |
-
try:
|
| 104 |
-
# if file not valid ou overwrite is enabled, move overwiting existing file
|
| 105 |
-
if not file_is_valid(self.final_path) or overwrite_if_valid:
|
| 106 |
-
os.makedirs(path.parent, exist_ok=True)
|
| 107 |
-
shutil.move(self.temp_file_path, path)
|
| 108 |
-
self.final_path = path
|
| 109 |
-
except Exception as e:
|
| 110 |
-
print(f"Error saving file: {e}")
|
| 111 |
-
return
|
| 112 |
-
|
| 113 |
-
# delete the file
|
| 114 |
-
def destroy(self):
|
| 115 |
-
try:
|
| 116 |
-
# destroy temporary file if it exists
|
| 117 |
-
if file_is_valid(self.temp_file_path):
|
| 118 |
-
os.remove(self.temp_file_path)
|
| 119 |
-
except Exception:
|
| 120 |
-
return
|
| 121 |
-
|
| 122 |
-
# copy an source file to destination if destination file is not equals to source
|
| 123 |
-
def copy_file_if_different(src_file: Path, dst_file: Path, silent: bool = False):
|
| 124 |
-
if file_is_valid(dst_file):
|
| 125 |
-
# Check if destination file exists and is different from source file
|
| 126 |
-
if filecmp.cmp(src_file, dst_file) and not silent:
|
| 127 |
-
print(f"{dst_file} already exists and is the same. No need to copy.")
|
| 128 |
-
return
|
| 129 |
-
|
| 130 |
-
os.makedirs(dst_file.parent, exist_ok=True)
|
| 131 |
-
shutil.copyfile(src_file, dst_file)
|
| 132 |
-
if not silent:
|
| 133 |
-
print(f"copied to {dst_file}")
|
| 134 |
-
|
| 135 |
-
# function to delete dir and all its content using shutil
|
| 136 |
-
def delete_folder(path: Path):
|
| 137 |
-
if path.is_dir():
|
| 138 |
-
shutil.rmtree(path)
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
def update_folder_times(folder_path):
|
| 142 |
-
folder_path = Path(folder_path)
|
| 143 |
-
|
| 144 |
-
# Keep track of the newest file's modification time for this folder
|
| 145 |
-
newest_file_time = None
|
| 146 |
-
|
| 147 |
-
for item in folder_path.iterdir():
|
| 148 |
-
# Check if the item is a file
|
| 149 |
-
if item.is_file():
|
| 150 |
-
file_time = item.stat().st_mtime # Get modification time of the file
|
| 151 |
-
|
| 152 |
-
# Update the newest_file_time if it's the first file or if the current file is newer
|
| 153 |
-
if newest_file_time is None or file_time > newest_file_time:
|
| 154 |
-
newest_file_time = file_time
|
| 155 |
-
|
| 156 |
-
# If the item is a subfolder, recursively update its times and find its newest file time
|
| 157 |
-
elif item.is_dir():
|
| 158 |
-
subfolder_newest_time = update_folder_times(item)
|
| 159 |
-
|
| 160 |
-
# Update the newest_file_time if a subfolder's newest file is newer
|
| 161 |
-
if newest_file_time is None or (subfolder_newest_time is not None and subfolder_newest_time > newest_file_time):
|
| 162 |
-
newest_file_time = subfolder_newest_time
|
| 163 |
-
|
| 164 |
-
# Update the folder's modification and creation times with the newest_file_time
|
| 165 |
-
if newest_file_time is not None:
|
| 166 |
-
# Convert to an integer (necessary on some systems)
|
| 167 |
-
newest_file_time = int(newest_file_time)
|
| 168 |
-
os.utime(path=folder_path, times=(newest_file_time, newest_file_time))
|
| 169 |
-
|
| 170 |
-
return newest_file_time # Return the newest_file_time to update the parent folder's time
|
|
|
|
| 1 |
+
import filecmp
|
| 2 |
+
import os
|
| 3 |
+
import shutil
|
| 4 |
+
import tempfile
|
| 5 |
+
from inspect import currentframe, getframeinfo
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
# return the itens in array that is not inexisting or empty
|
| 9 |
+
def validate_files(paths):
|
| 10 |
+
valid_files = [path for path in paths if file_is_valid(path)]
|
| 11 |
+
return valid_files
|
| 12 |
+
|
| 13 |
+
# check if a file is existing and not empty
|
| 14 |
+
def file_is_valid(path):
|
| 15 |
+
if path is not None and path.is_file():
|
| 16 |
+
size = path.stat().st_size
|
| 17 |
+
if size > 0:
|
| 18 |
+
return True
|
| 19 |
+
return False
|
| 20 |
+
|
| 21 |
+
# validate if an string is a valid dir or path with valid content
|
| 22 |
+
def check_valid_path(path_str):
|
| 23 |
+
# create a Path object from the input string
|
| 24 |
+
path = Path(path_str)
|
| 25 |
+
|
| 26 |
+
# check if the path exists
|
| 27 |
+
if not path.exists():
|
| 28 |
+
raise FileNotFoundError(f"The path '{path_str}' does not exist.")
|
| 29 |
+
|
| 30 |
+
# check if it's a directory with at least one file or a valid file with content
|
| 31 |
+
if path.is_dir():
|
| 32 |
+
# check if the directory has at least one file with content
|
| 33 |
+
files_with_content = [file for file in path.iterdir() if file.is_file() and file.stat().st_size > 0]
|
| 34 |
+
if not files_with_content:
|
| 35 |
+
raise ValueError(f"The directory '{path_str}' does not contain any files with content.")
|
| 36 |
+
elif path.is_file():
|
| 37 |
+
# check if the file has content
|
| 38 |
+
if path.stat().st_size == 0:
|
| 39 |
+
raise ValueError(f"The file '{path_str}' does not contain any content.")
|
| 40 |
+
else:
|
| 41 |
+
raise ValueError(f"The path '{path_str}' is neither a valid directory nor a valid file.")
|
| 42 |
+
|
| 43 |
+
return path
|
| 44 |
+
|
| 45 |
+
# validate if an string is a valid dir or path with valid content
|
| 46 |
+
def check_existing_path(path_str):
|
| 47 |
+
# create a Path object from the input string
|
| 48 |
+
path = Path(path_str)
|
| 49 |
+
|
| 50 |
+
# check if the path exists
|
| 51 |
+
if not path.exists():
|
| 52 |
+
raise FileNotFoundError(f"The path '{path_str}' does not exist.")
|
| 53 |
+
|
| 54 |
+
# check if it's a directory with at least one file or a valid file with content
|
| 55 |
+
if not path.is_dir() and not path.is_file():
|
| 56 |
+
raise ValueError(f"The path '{path_str}' is neither a valid directory nor a valid file.")
|
| 57 |
+
|
| 58 |
+
return path_str
|
| 59 |
+
|
| 60 |
+
# create a tempfile class to use as object
|
| 61 |
+
class TempFile:
|
| 62 |
+
|
| 63 |
+
def __init__(self, final_path: Path, file_ext: str = None):
|
| 64 |
+
self.final_path: Path = None if final_path is None else Path(
|
| 65 |
+
final_path)
|
| 66 |
+
self.file_ext = file_ext
|
| 67 |
+
os.makedirs(Path(Path(getframeinfo(
|
| 68 |
+
currentframe()).filename).resolve().parent, "temp"), exist_ok=True)
|
| 69 |
+
self.temp_file: tempfile.NamedTemporaryFile = tempfile.NamedTemporaryFile(dir=Path(Path(getframeinfo(currentframe()).filename).resolve().parent, "temp"),
|
| 70 |
+
delete=False, suffix=file_ext)
|
| 71 |
+
|
| 72 |
+
self.temp_file_name = self.temp_file.name
|
| 73 |
+
self.temp_file_path: Path = Path(self.temp_file.name)
|
| 74 |
+
|
| 75 |
+
self.temp_file.close()
|
| 76 |
+
|
| 77 |
+
# return the actual path of the file
|
| 78 |
+
def getpath(self):
|
| 79 |
+
if self.temp_file_path.is_file():
|
| 80 |
+
return self.temp_file_path
|
| 81 |
+
elif file_is_valid(self.final_path):
|
| 82 |
+
return self.final_path
|
| 83 |
+
else:
|
| 84 |
+
return None
|
| 85 |
+
|
| 86 |
+
# return the actual path of the file if not empty
|
| 87 |
+
def getvalidpath(self):
|
| 88 |
+
if file_is_valid(self.temp_file_path):
|
| 89 |
+
return self.temp_file_path
|
| 90 |
+
elif file_is_valid(self.final_path):
|
| 91 |
+
return self.final_path
|
| 92 |
+
else:
|
| 93 |
+
return None
|
| 94 |
+
|
| 95 |
+
# save the temp file into final path
|
| 96 |
+
def save(self, overwrite_if_valid: bool = True, update_path: Path = None):
|
| 97 |
+
# update final path case user specifies it
|
| 98 |
+
if update_path is None:
|
| 99 |
+
path: Path = self.final_path
|
| 100 |
+
else:
|
| 101 |
+
path: Path = update_path
|
| 102 |
+
|
| 103 |
+
try:
|
| 104 |
+
# if file not valid ou overwrite is enabled, move overwiting existing file
|
| 105 |
+
if not file_is_valid(self.final_path) or overwrite_if_valid:
|
| 106 |
+
os.makedirs(path.parent, exist_ok=True)
|
| 107 |
+
shutil.move(self.temp_file_path, path)
|
| 108 |
+
self.final_path = path
|
| 109 |
+
except Exception as e:
|
| 110 |
+
print(f"Error saving file: {e}")
|
| 111 |
+
return
|
| 112 |
+
|
| 113 |
+
# delete the file
|
| 114 |
+
def destroy(self):
|
| 115 |
+
try:
|
| 116 |
+
# destroy temporary file if it exists
|
| 117 |
+
if file_is_valid(self.temp_file_path):
|
| 118 |
+
os.remove(self.temp_file_path)
|
| 119 |
+
except Exception:
|
| 120 |
+
return
|
| 121 |
+
|
| 122 |
+
# copy an source file to destination if destination file is not equals to source
|
| 123 |
+
def copy_file_if_different(src_file: Path, dst_file: Path, silent: bool = False):
|
| 124 |
+
if file_is_valid(dst_file):
|
| 125 |
+
# Check if destination file exists and is different from source file
|
| 126 |
+
if filecmp.cmp(src_file, dst_file) and not silent:
|
| 127 |
+
print(f"{dst_file} already exists and is the same. No need to copy.")
|
| 128 |
+
return
|
| 129 |
+
|
| 130 |
+
os.makedirs(dst_file.parent, exist_ok=True)
|
| 131 |
+
shutil.copyfile(src_file, dst_file)
|
| 132 |
+
if not silent:
|
| 133 |
+
print(f"copied to {dst_file}")
|
| 134 |
+
|
| 135 |
+
# function to delete dir and all its content using shutil
|
| 136 |
+
def delete_folder(path: Path):
|
| 137 |
+
if path.is_dir():
|
| 138 |
+
shutil.rmtree(path)
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def update_folder_times(folder_path):
|
| 142 |
+
folder_path = Path(folder_path)
|
| 143 |
+
|
| 144 |
+
# Keep track of the newest file's modification time for this folder
|
| 145 |
+
newest_file_time = None
|
| 146 |
+
|
| 147 |
+
for item in folder_path.iterdir():
|
| 148 |
+
# Check if the item is a file
|
| 149 |
+
if item.is_file():
|
| 150 |
+
file_time = item.stat().st_mtime # Get modification time of the file
|
| 151 |
+
|
| 152 |
+
# Update the newest_file_time if it's the first file or if the current file is newer
|
| 153 |
+
if newest_file_time is None or file_time > newest_file_time:
|
| 154 |
+
newest_file_time = file_time
|
| 155 |
+
|
| 156 |
+
# If the item is a subfolder, recursively update its times and find its newest file time
|
| 157 |
+
elif item.is_dir():
|
| 158 |
+
subfolder_newest_time = update_folder_times(item)
|
| 159 |
+
|
| 160 |
+
# Update the newest_file_time if a subfolder's newest file is newer
|
| 161 |
+
if newest_file_time is None or (subfolder_newest_time is not None and subfolder_newest_time > newest_file_time):
|
| 162 |
+
newest_file_time = subfolder_newest_time
|
| 163 |
+
|
| 164 |
+
# Update the folder's modification and creation times with the newest_file_time
|
| 165 |
+
if newest_file_time is not None:
|
| 166 |
+
# Convert to an integer (necessary on some systems)
|
| 167 |
+
newest_file_time = int(newest_file_time)
|
| 168 |
+
os.utime(path=folder_path, times=(newest_file_time, newest_file_time))
|
| 169 |
+
|
| 170 |
+
return newest_file_time # Return the newest_file_time to update the parent folder's time
|
gemini_utils.py
ADDED
|
@@ -0,0 +1,840 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
import math
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
import sys
|
| 7 |
+
import threading
|
| 8 |
+
import time
|
| 9 |
+
from typing import Callable, Iterable, List, Sequence
|
| 10 |
+
|
| 11 |
+
import pysrt
|
| 12 |
+
|
| 13 |
+
import gemini_srt_translator as gst
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@dataclass(frozen=True)
|
| 17 |
+
class GeminiTranslationConfig:
|
| 18 |
+
api_keys: Sequence[str]
|
| 19 |
+
input_file: Path
|
| 20 |
+
output_file: Path
|
| 21 |
+
target_language: str
|
| 22 |
+
batch_size: int = 500
|
| 23 |
+
temperature: float = 0.3
|
| 24 |
+
top_p: float = 0.9
|
| 25 |
+
top_k: int = 50
|
| 26 |
+
free_quota: bool = True
|
| 27 |
+
resume: bool = False
|
| 28 |
+
thinking: bool = True
|
| 29 |
+
progress_log: bool = False
|
| 30 |
+
thoughts_log: bool = False
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class MultiKeyGeminiTranslator(gst.GeminiSRTTranslator):
|
| 34 |
+
"""Gemini translator that rotates across an arbitrary number of API keys."""
|
| 35 |
+
|
| 36 |
+
def __init__(self, api_keys: Sequence[str], **kwargs) -> None:
|
| 37 |
+
cleaned: List[str] = [key.strip() for key in api_keys if key and key.strip()]
|
| 38 |
+
if not cleaned:
|
| 39 |
+
raise ValueError("At least one Gemini API key is required.")
|
| 40 |
+
|
| 41 |
+
primary = cleaned[0]
|
| 42 |
+
secondary = cleaned[1] if len(cleaned) > 1 else None
|
| 43 |
+
|
| 44 |
+
super().__init__(gemini_api_key=primary, gemini_api_key2=secondary, **kwargs)
|
| 45 |
+
|
| 46 |
+
self._api_keys = cleaned
|
| 47 |
+
self._api_index = 0
|
| 48 |
+
self.current_api_key = primary
|
| 49 |
+
self.current_api_number = 1
|
| 50 |
+
self.backup_api_number = 2 if len(cleaned) > 1 else 1
|
| 51 |
+
|
| 52 |
+
def _switch_api(self) -> bool: # type: ignore[override]
|
| 53 |
+
if len(self._api_keys) <= 1:
|
| 54 |
+
return False
|
| 55 |
+
|
| 56 |
+
previous_number = self.current_api_number
|
| 57 |
+
total_keys = len(self._api_keys)
|
| 58 |
+
|
| 59 |
+
for step in range(1, total_keys + 1):
|
| 60 |
+
next_index = (self._api_index + step) % total_keys
|
| 61 |
+
if next_index == self._api_index:
|
| 62 |
+
continue
|
| 63 |
+
|
| 64 |
+
next_key = self._api_keys[next_index]
|
| 65 |
+
if next_key:
|
| 66 |
+
self._api_index = next_index
|
| 67 |
+
self.current_api_key = next_key
|
| 68 |
+
self.current_api_number = next_index + 1
|
| 69 |
+
self.backup_api_number = previous_number
|
| 70 |
+
return True
|
| 71 |
+
|
| 72 |
+
return False
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def translate_with_gemini(config: GeminiTranslationConfig) -> pysrt.SubRipFile:
|
| 76 |
+
additional_instructions = (
|
| 77 |
+
"CRITICAL INSTRUCTIONS:\n"
|
| 78 |
+
"1. You MUST return exactly the same number of objects as the input batch.\n"
|
| 79 |
+
"2. Check the input segments count and ensure your output count matches exactly.\n"
|
| 80 |
+
"3. Do not skip any index. Every input object must have a corresponding output object.\n"
|
| 81 |
+
"4. If a line is empty in input, keep it empty in output.\n"
|
| 82 |
+
"5. If a line has content, it MUST be translated. Do not return empty strings for non-empty input.\n"
|
| 83 |
+
"6. Do not merge or split subtitles.\n"
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
translator = MultiKeyGeminiTranslator(
|
| 87 |
+
api_keys=config.api_keys,
|
| 88 |
+
target_language=config.target_language,
|
| 89 |
+
input_file=str(config.input_file),
|
| 90 |
+
output_file=str(config.output_file),
|
| 91 |
+
batch_size=config.batch_size,
|
| 92 |
+
temperature=config.temperature,
|
| 93 |
+
top_p=config.top_p,
|
| 94 |
+
top_k=config.top_k,
|
| 95 |
+
free_quota=config.free_quota,
|
| 96 |
+
resume=config.resume,
|
| 97 |
+
thinking=config.thinking,
|
| 98 |
+
progress_log=config.progress_log,
|
| 99 |
+
thoughts_log=config.thoughts_log,
|
| 100 |
+
description=additional_instructions,
|
| 101 |
+
)
|
| 102 |
+
|
| 103 |
+
translator.translate()
|
| 104 |
+
|
| 105 |
+
return pysrt.open(config.output_file, encoding="utf-8")
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def normalize_api_keys(keys: Iterable[str] | str | None) -> List[str]:
|
| 109 |
+
if keys is None:
|
| 110 |
+
return []
|
| 111 |
+
|
| 112 |
+
if isinstance(keys, str):
|
| 113 |
+
raw = [keys]
|
| 114 |
+
else:
|
| 115 |
+
raw = list(keys)
|
| 116 |
+
|
| 117 |
+
candidates: List[str] = []
|
| 118 |
+
for value in raw:
|
| 119 |
+
if not value:
|
| 120 |
+
continue
|
| 121 |
+
parts = [part.strip() for part in str(value).replace("\n", ",").split(",")]
|
| 122 |
+
candidates.extend(part for part in parts if part)
|
| 123 |
+
|
| 124 |
+
unique: List[str] = []
|
| 125 |
+
seen = set()
|
| 126 |
+
for key in candidates:
|
| 127 |
+
if key not in seen:
|
| 128 |
+
unique.append(key)
|
| 129 |
+
seen.add(key)
|
| 130 |
+
|
| 131 |
+
return unique
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
@dataclass(frozen=True)
|
| 135 |
+
class GeminiSummaryConfig:
|
| 136 |
+
api_keys: Sequence[str]
|
| 137 |
+
subtitle_file: Path
|
| 138 |
+
output_file: Path
|
| 139 |
+
language: str
|
| 140 |
+
model: str = "gemini-2.5-flash"
|
| 141 |
+
# TLTW tends to be more useful with longer outputs. If the model truncates,
|
| 142 |
+
# we auto-continue (see _send_tltw_request).
|
| 143 |
+
max_output_tokens: int = 16364
|
| 144 |
+
final_max_output_tokens: int | None = None
|
| 145 |
+
request_timeout: int = 500
|
| 146 |
+
truncate_chars: int | None = None
|
| 147 |
+
# With modern Gemini context windows, chunking is often unnecessary and can
|
| 148 |
+
# reduce coherence. Keep it disabled by default; enable explicitly for very
|
| 149 |
+
# large inputs or when you want extra robustness.
|
| 150 |
+
chunk_chars: int | None = None
|
| 151 |
+
|
| 152 |
+
# Inference tuning (lower temperature => more stable outputs)
|
| 153 |
+
temperature: float = 0.15
|
| 154 |
+
top_p: float = 0.9
|
| 155 |
+
top_k: int = 40
|
| 156 |
+
|
| 157 |
+
# Auto-continue when the model hits output limits
|
| 158 |
+
max_rounds: int = 10
|
| 159 |
+
continuation_tail_chars: int = 800
|
| 160 |
+
|
| 161 |
+
# CLI/console progress
|
| 162 |
+
show_progress: bool = True
|
| 163 |
+
progress_update_interval: float = 0.5
|
| 164 |
+
progress_preview_chars: int = 80
|
| 165 |
+
stream_output: bool = True
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def _mask_api_key(key: str) -> str:
|
| 169 |
+
cleaned = (key or "").strip()
|
| 170 |
+
if len(cleaned) <= 6:
|
| 171 |
+
return "***"
|
| 172 |
+
return "***" + cleaned[-4:]
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def _strip_ansi(text: str) -> str:
|
| 176 |
+
# Minimal ANSI stripper to keep progress lines tidy in some terminals.
|
| 177 |
+
out = []
|
| 178 |
+
i = 0
|
| 179 |
+
while i < len(text):
|
| 180 |
+
ch = text[i]
|
| 181 |
+
if ch == "\x1b":
|
| 182 |
+
# Skip CSI sequences.
|
| 183 |
+
while i < len(text) and text[i] != "m":
|
| 184 |
+
i += 1
|
| 185 |
+
i += 1
|
| 186 |
+
continue
|
| 187 |
+
out.append(ch)
|
| 188 |
+
i += 1
|
| 189 |
+
return "".join(out)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def _single_line_preview(text: str, limit: int) -> str:
|
| 193 |
+
cleaned = " ".join((text or "").split())
|
| 194 |
+
if limit <= 0:
|
| 195 |
+
return ""
|
| 196 |
+
if len(cleaned) <= limit:
|
| 197 |
+
return cleaned
|
| 198 |
+
return cleaned[: max(0, limit - 1)] + "…"
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
class _ProgressTicker:
|
| 202 |
+
def __init__(
|
| 203 |
+
self,
|
| 204 |
+
*,
|
| 205 |
+
label: str,
|
| 206 |
+
expected_seconds: float,
|
| 207 |
+
update_interval: float,
|
| 208 |
+
preview_supplier: Callable[[], str],
|
| 209 |
+
) -> None:
|
| 210 |
+
self._label = label
|
| 211 |
+
self._expected_seconds = max(1.0, float(expected_seconds))
|
| 212 |
+
self._update_interval = max(0.05, float(update_interval))
|
| 213 |
+
self._preview_supplier = preview_supplier
|
| 214 |
+
self._stop = threading.Event()
|
| 215 |
+
self._thread: threading.Thread | None = None
|
| 216 |
+
self._start = 0.0
|
| 217 |
+
|
| 218 |
+
def start(self) -> None:
|
| 219 |
+
if not sys.stderr.isatty():
|
| 220 |
+
return
|
| 221 |
+
self._start = time.time()
|
| 222 |
+
self._thread = threading.Thread(target=self._run, daemon=True)
|
| 223 |
+
self._thread.start()
|
| 224 |
+
|
| 225 |
+
def stop(self) -> None:
|
| 226 |
+
if not sys.stderr.isatty():
|
| 227 |
+
return
|
| 228 |
+
self._stop.set()
|
| 229 |
+
if self._thread is not None:
|
| 230 |
+
self._thread.join(timeout=1.0)
|
| 231 |
+
|
| 232 |
+
def _run(self) -> None:
|
| 233 |
+
# tqdm is already a dependency of this project (requirements.txt).
|
| 234 |
+
from tqdm import tqdm
|
| 235 |
+
|
| 236 |
+
# We intentionally display ONLY the percentage (no bar), plus an optional postfix.
|
| 237 |
+
pbar = tqdm(
|
| 238 |
+
total=100,
|
| 239 |
+
desc=self._label,
|
| 240 |
+
file=sys.stderr,
|
| 241 |
+
leave=False,
|
| 242 |
+
dynamic_ncols=True,
|
| 243 |
+
bar_format="{desc} {percentage:3.0f}% | {postfix}",
|
| 244 |
+
)
|
| 245 |
+
|
| 246 |
+
try:
|
| 247 |
+
last_n = 0
|
| 248 |
+
while not self._stop.is_set():
|
| 249 |
+
elapsed = max(0.0, time.time() - self._start)
|
| 250 |
+
pct = min(0.99, elapsed / self._expected_seconds)
|
| 251 |
+
target_n = int(pct * 100)
|
| 252 |
+
if target_n > last_n:
|
| 253 |
+
pbar.update(target_n - last_n)
|
| 254 |
+
last_n = target_n
|
| 255 |
+
|
| 256 |
+
preview = self._preview_supplier() if self._preview_supplier else ""
|
| 257 |
+
preview = _strip_ansi(preview)
|
| 258 |
+
preview = _single_line_preview(preview, 80)
|
| 259 |
+
pbar.set_postfix_str(preview)
|
| 260 |
+
|
| 261 |
+
pbar.refresh()
|
| 262 |
+
time.sleep(self._update_interval)
|
| 263 |
+
|
| 264 |
+
# Finish to 100% on stop.
|
| 265 |
+
if last_n < 100:
|
| 266 |
+
pbar.update(100 - last_n)
|
| 267 |
+
pbar.refresh()
|
| 268 |
+
finally:
|
| 269 |
+
pbar.close()
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def _load_srt_as_text(subtitle_file: Path, truncate_chars: int | None) -> str:
|
| 273 |
+
"""Read the subtitle file as raw text, optionally truncating for safety."""
|
| 274 |
+
|
| 275 |
+
content = subtitle_file.read_text(encoding="utf-8", errors="replace").strip()
|
| 276 |
+
if truncate_chars and truncate_chars > 0 and len(content) > truncate_chars:
|
| 277 |
+
return content[:truncate_chars]
|
| 278 |
+
return content
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def _build_tltw_prompt(language: str) -> str:
|
| 282 |
+
# Backward-compatible default prompt (duration-aware prompt is built in generate_tltw_summary).
|
| 283 |
+
return _build_tltw_prompt_with_limits(language=language, max_key_points=12)
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def _build_tltw_prompt_with_limits(*, language: str, max_key_points: int, min_key_points: int | None = None) -> str:
|
| 287 |
+
if max_key_points < 1:
|
| 288 |
+
max_key_points = 1
|
| 289 |
+
if min_key_points is None:
|
| 290 |
+
min_key_points = max(1, int(math.floor(max_key_points * 0.6)))
|
| 291 |
+
else:
|
| 292 |
+
min_key_points = max(1, min(int(min_key_points), int(max_key_points)))
|
| 293 |
+
|
| 294 |
+
return (
|
| 295 |
+
"Generate a high-quality 'TLTW' (Too Long To Watch) summary in "
|
| 296 |
+
f"{language} strictly based on the provided transcript or subtitles. "
|
| 297 |
+
"The TLTW must be factual, concise, and faithful to the source.\n\n"
|
| 298 |
+
|
| 299 |
+
"Use Markdown and follow this exact structure:\n\n"
|
| 300 |
+
|
| 301 |
+
"# Title\n"
|
| 302 |
+
"- A clear, descriptive title reflecting the main subject of the content\n\n"
|
| 303 |
+
|
| 304 |
+
"*Tags: [tag 1, tag 2, ...]*\n"
|
| 305 |
+
"- 3–9 short tags, comma-separated, singular words when possible, multi-word-expressions only when significant, lowercase when possible (e.g., `ai, subtitles, whisperx, translation`)\n"
|
| 306 |
+
"- Tags must reflect topics actually present in the source\n\n"
|
| 307 |
+
|
| 308 |
+
"## Key Points\n"
|
| 309 |
+
f"- Write at most {max_key_points} bullet points\n"
|
| 310 |
+
f"- Prefer {min_key_points}–{max_key_points} bullet points; you may go up to 40% smaller to avoid redundancy\n"
|
| 311 |
+
"- Put the most important points first\n"
|
| 312 |
+
"- Preserve technical terminology used in the source when relevant\n"
|
| 313 |
+
"- Each bullet must briefly explain the point (topic + a short clarifying clause), not just label it\n"
|
| 314 |
+
"- Avoid generic takeaways like 'the importance of', 'the need of', 'is crucial', unless tied to a specific situation described\n"
|
| 315 |
+
"- Avoid advice/recommendations here; put them only in Actions when explicitly present\n"
|
| 316 |
+
"- Do not end bullet lines with a period (no trailing '.')\n\n"
|
| 317 |
+
|
| 318 |
+
"## Actions or Next Steps (only if applicable)\n"
|
| 319 |
+
"- Bullet points describing explicit recommendations, procedures, or follow-ups mentioned in the source\n"
|
| 320 |
+
"- Omit this section entirely if no actionable items are present\n"
|
| 321 |
+
"- Do not end bullet lines with a period (no trailing '.')\n\n"
|
| 322 |
+
|
| 323 |
+
"## Summary\n"
|
| 324 |
+
f"- Write at most {max_key_points} chapter titles\n"
|
| 325 |
+
f"- Prefer {min_key_points}–{max_key_points} chapter titles; you may go up to 40% smaller to avoid redundancy\n"
|
| 326 |
+
"- Each title must be 1 short sentence\n"
|
| 327 |
+
"- Write chapter-like titles, not takeaways: describe the topic, not what the viewer should learn/do\n"
|
| 328 |
+
"- Prefer noun phrases and topic labels; avoid advice, conclusions, recommendations, or moral-of-the-story phrasing\n"
|
| 329 |
+
"- Suppress authorship; write each line like a book chapter title, not about who said/did it\n"
|
| 330 |
+
"- Each title must be a single line in the format: HH:MM:SS description\n"
|
| 331 |
+
"- Use HH:MM:SS (no milliseconds)\n"
|
| 332 |
+
"- Keep items in chronological order\n"
|
| 333 |
+
"- Use timestamps aligned to the subtitle timeline (best possible approximation based on nearby lines)\n"
|
| 334 |
+
"- Do not use bullet markers for these lines\n"
|
| 335 |
+
"- Do not end lines with a period (no trailing '.')\n\n"
|
| 336 |
+
"Example format:\n"
|
| 337 |
+
"```\n"
|
| 338 |
+
"00:00:02 chapter title description\n"
|
| 339 |
+
"00:22:13 chapter title description\n"
|
| 340 |
+
"```\n\n"
|
| 341 |
+
|
| 342 |
+
"Writing style requirements:\n"
|
| 343 |
+
"- Use active voice; avoid passive constructions\n"
|
| 344 |
+
"- Be concise; remove filler words\n"
|
| 345 |
+
"- Use an assertive, direct tone\n"
|
| 346 |
+
"- Start lines with an article only when it naturally fits; prefer direct noun-phrase titles and topic labels\n\n"
|
| 347 |
+
|
| 348 |
+
"Rules:\n"
|
| 349 |
+
f"- Write exclusively in {language}\n"
|
| 350 |
+
"- Do not invent, extrapolate, or assume information\n"
|
| 351 |
+
"- Avoid redundancy and meta commentary\n"
|
| 352 |
+
"- Keep sentences short, direct, and information-dense\n"
|
| 353 |
+
"- Dont cause repetition of structures\n"
|
| 354 |
+
"- Do not apologize or reference missing context\n"
|
| 355 |
+
"- Do not mention the transcript or the act of summarization\n"
|
| 356 |
+
"- End the document with a final line exactly equal to: <!-- END -->\n"
|
| 357 |
+
)
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def _estimate_srt_duration_seconds(subtitle_file: Path) -> float:
|
| 361 |
+
"""Best-effort estimate of the subtitle duration in seconds."""
|
| 362 |
+
|
| 363 |
+
try:
|
| 364 |
+
subs = pysrt.open(subtitle_file, encoding="utf-8")
|
| 365 |
+
except Exception:
|
| 366 |
+
return 0.0
|
| 367 |
+
if not subs:
|
| 368 |
+
return 0.0
|
| 369 |
+
try:
|
| 370 |
+
last_end_ms = subs[-1].end.ordinal
|
| 371 |
+
return max(0.0, float(last_end_ms) / 1000.0)
|
| 372 |
+
except Exception:
|
| 373 |
+
return 0.0
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
def _build_chunk_prompt(language: str, chunk_index: int, chunk_total: int) -> str:
|
| 378 |
+
return (
|
| 379 |
+
f"You are summarizing chunk {chunk_index}/{chunk_total} of a transcript. "
|
| 380 |
+
f"Write in {language}, strictly based on this chunk. "
|
| 381 |
+
"Return Markdown with exactly one section: '## Key Points' followed by bullet points. "
|
| 382 |
+
"Do not include a title or any other sections. "
|
| 383 |
+
"Each bullet must include a concrete anchor from this chunk and a brief explanation (topic + short clarifying clause). "
|
| 384 |
+
"Avoid generic takeaways and avoid advice unless explicitly present. "
|
| 385 |
+
"Use active voice, be concise, and suppress authorship (chapter-title style). "
|
| 386 |
+
"Do not end bullet lines with a period (no trailing '.')."
|
| 387 |
+
)
|
| 388 |
+
|
| 389 |
+
|
| 390 |
+
def _send_tltw_request(
|
| 391 |
+
*,
|
| 392 |
+
api_key: str,
|
| 393 |
+
subtitle_text: str,
|
| 394 |
+
language: str,
|
| 395 |
+
model: str,
|
| 396 |
+
max_output_tokens: int,
|
| 397 |
+
request_timeout: int,
|
| 398 |
+
prompt_builder: Callable[[str], str] = _build_tltw_prompt,
|
| 399 |
+
temperature: float = 0.2,
|
| 400 |
+
top_p: float = 0.9,
|
| 401 |
+
top_k: int = 40,
|
| 402 |
+
max_rounds: int = 3,
|
| 403 |
+
continuation_tail_chars: int = 800,
|
| 404 |
+
show_progress: bool = False,
|
| 405 |
+
progress_update_interval: float = 0.2,
|
| 406 |
+
progress_preview_chars: int = 140,
|
| 407 |
+
stream_output: bool = False,
|
| 408 |
+
) -> str:
|
| 409 |
+
try:
|
| 410 |
+
import google.generativeai as genai
|
| 411 |
+
except Exception as exc: # pragma: no cover - import depends on optional dependency
|
| 412 |
+
raise RuntimeError("google-generativeai is required for TLTW summaries.") from exc
|
| 413 |
+
|
| 414 |
+
genai.configure(api_key=api_key)
|
| 415 |
+
|
| 416 |
+
END_MARKER = "<!-- END -->"
|
| 417 |
+
base_prompt = prompt_builder(language)
|
| 418 |
+
require_end_marker = END_MARKER in base_prompt
|
| 419 |
+
model_client = genai.GenerativeModel(model)
|
| 420 |
+
|
| 421 |
+
def _finish_reason_is_truncation(resp) -> bool:
|
| 422 |
+
try:
|
| 423 |
+
candidates = getattr(resp, "candidates", None) or []
|
| 424 |
+
if not candidates:
|
| 425 |
+
return False
|
| 426 |
+
finish_reason = getattr(candidates[0], "finish_reason", None)
|
| 427 |
+
if finish_reason is None:
|
| 428 |
+
return False
|
| 429 |
+
finish_str = str(finish_reason).lower()
|
| 430 |
+
return "max" in finish_str or "token" in finish_str
|
| 431 |
+
except Exception:
|
| 432 |
+
return False
|
| 433 |
+
|
| 434 |
+
last_preview: str = ""
|
| 435 |
+
progress_lines_count: int = 0
|
| 436 |
+
|
| 437 |
+
def _expected_seconds_for_call(*, include_subtitle_text: bool) -> float:
|
| 438 |
+
# Heuristic only. Goal: a progress bar that feels realistic.
|
| 439 |
+
base = 2.0
|
| 440 |
+
input_factor = (len(subtitle_text) / 50_000.0) * (3.0 if include_subtitle_text else 0.8)
|
| 441 |
+
output_factor = (max_output_tokens / 2000.0) * 1.8
|
| 442 |
+
return base + input_factor + output_factor
|
| 443 |
+
|
| 444 |
+
def _generate(
|
| 445 |
+
prompt: str,
|
| 446 |
+
*,
|
| 447 |
+
include_subtitle_text: bool,
|
| 448 |
+
phase_label: str,
|
| 449 |
+
base_lines: int,
|
| 450 |
+
) -> tuple[str, bool]:
|
| 451 |
+
nonlocal last_preview
|
| 452 |
+
nonlocal progress_lines_count
|
| 453 |
+
|
| 454 |
+
parts = [prompt]
|
| 455 |
+
if include_subtitle_text:
|
| 456 |
+
parts.append(subtitle_text)
|
| 457 |
+
|
| 458 |
+
tqdm_active = bool(show_progress and sys.stderr.isatty())
|
| 459 |
+
ticker: _ProgressTicker | None = None
|
| 460 |
+
progress_lines_count = max(0, int(base_lines))
|
| 461 |
+
|
| 462 |
+
# Prefer streaming text preview when available; otherwise show line counts.
|
| 463 |
+
use_text_preview = bool(stream_output and tqdm_active)
|
| 464 |
+
emit_stream_to_stderr = bool(stream_output and show_progress and not tqdm_active)
|
| 465 |
+
|
| 466 |
+
if show_progress:
|
| 467 |
+
expected = _expected_seconds_for_call(include_subtitle_text=include_subtitle_text)
|
| 468 |
+
ticker = _ProgressTicker(
|
| 469 |
+
label=phase_label,
|
| 470 |
+
expected_seconds=expected,
|
| 471 |
+
update_interval=progress_update_interval,
|
| 472 |
+
preview_supplier=lambda: (
|
| 473 |
+
last_preview if use_text_preview else f"already generated {progress_lines_count} lines"
|
| 474 |
+
),
|
| 475 |
+
)
|
| 476 |
+
if tqdm_active:
|
| 477 |
+
ticker.start()
|
| 478 |
+
else:
|
| 479 |
+
sys.stderr.write(f"{phase_label}...\n")
|
| 480 |
+
sys.stderr.flush()
|
| 481 |
+
|
| 482 |
+
response = None
|
| 483 |
+
text = ""
|
| 484 |
+
|
| 485 |
+
try:
|
| 486 |
+
gen_kwargs = dict(
|
| 487 |
+
generation_config={
|
| 488 |
+
"temperature": float(temperature),
|
| 489 |
+
"top_p": float(top_p),
|
| 490 |
+
"top_k": int(top_k),
|
| 491 |
+
"max_output_tokens": int(max_output_tokens),
|
| 492 |
+
},
|
| 493 |
+
request_options={"timeout": request_timeout},
|
| 494 |
+
)
|
| 495 |
+
|
| 496 |
+
if stream_output:
|
| 497 |
+
try:
|
| 498 |
+
response_iter = model_client.generate_content(parts, stream=True, **gen_kwargs)
|
| 499 |
+
acc = ""
|
| 500 |
+
emitted_len = 0
|
| 501 |
+
last_resp = None
|
| 502 |
+
for resp in response_iter:
|
| 503 |
+
last_resp = resp
|
| 504 |
+
chunk_text = (getattr(resp, "text", "") or "")
|
| 505 |
+
if not chunk_text:
|
| 506 |
+
continue
|
| 507 |
+
if chunk_text.startswith(acc):
|
| 508 |
+
acc = chunk_text
|
| 509 |
+
else:
|
| 510 |
+
acc += chunk_text
|
| 511 |
+
|
| 512 |
+
if emit_stream_to_stderr and len(acc) > emitted_len:
|
| 513 |
+
sys.stderr.write(acc[emitted_len:])
|
| 514 |
+
sys.stderr.flush()
|
| 515 |
+
emitted_len = len(acc)
|
| 516 |
+
|
| 517 |
+
# Update progress as text streams in.
|
| 518 |
+
progress_lines_count = base_lines + (acc.count("\n") + (1 if acc else 0))
|
| 519 |
+
last_preview = _single_line_preview(
|
| 520 |
+
acc[-max(0, progress_preview_chars * 3) :],
|
| 521 |
+
progress_preview_chars,
|
| 522 |
+
)
|
| 523 |
+
|
| 524 |
+
response = last_resp
|
| 525 |
+
text = acc.strip()
|
| 526 |
+
|
| 527 |
+
if emit_stream_to_stderr and acc and not acc.endswith("\n"):
|
| 528 |
+
sys.stderr.write("\n")
|
| 529 |
+
sys.stderr.flush()
|
| 530 |
+
except Exception:
|
| 531 |
+
# Some environments/models may not support streaming.
|
| 532 |
+
use_text_preview = False
|
| 533 |
+
response = model_client.generate_content(parts, **gen_kwargs)
|
| 534 |
+
text = (getattr(response, "text", "") or "").strip()
|
| 535 |
+
else:
|
| 536 |
+
use_text_preview = False
|
| 537 |
+
response = model_client.generate_content(parts, **gen_kwargs)
|
| 538 |
+
text = (getattr(response, "text", "") or "").strip()
|
| 539 |
+
except Exception as exc: # noqa: BLE001
|
| 540 |
+
if ticker is not None:
|
| 541 |
+
ticker.stop()
|
| 542 |
+
if show_progress and not tqdm_active:
|
| 543 |
+
sys.stderr.write(f"{phase_label} failed: {exc}\n")
|
| 544 |
+
sys.stderr.flush()
|
| 545 |
+
raise
|
| 546 |
+
finally:
|
| 547 |
+
if ticker is not None:
|
| 548 |
+
ticker.stop()
|
| 549 |
+
|
| 550 |
+
if not text:
|
| 551 |
+
if show_progress and not tqdm_active:
|
| 552 |
+
sys.stderr.write(f"{phase_label} failed: empty response\n")
|
| 553 |
+
sys.stderr.flush()
|
| 554 |
+
raise RuntimeError("Empty response from Gemini while generating TLTW summary.")
|
| 555 |
+
|
| 556 |
+
progress_lines_count = base_lines + (text.count("\n") + 1)
|
| 557 |
+
last_preview = _single_line_preview(text[-max(0, progress_preview_chars * 2) :], progress_preview_chars)
|
| 558 |
+
|
| 559 |
+
truncated = _finish_reason_is_truncation(response)
|
| 560 |
+
if require_end_marker and END_MARKER not in text:
|
| 561 |
+
truncated = True
|
| 562 |
+
|
| 563 |
+
if show_progress and not tqdm_active:
|
| 564 |
+
sys.stderr.write(f"{phase_label} done\n")
|
| 565 |
+
sys.stderr.flush()
|
| 566 |
+
|
| 567 |
+
return text, truncated
|
| 568 |
+
|
| 569 |
+
if show_progress and not sys.stderr.isatty():
|
| 570 |
+
sys.stderr.write(
|
| 571 |
+
f"TLTW request: model={model} key={_mask_api_key(api_key)} max_output_tokens={max_output_tokens} timeout={request_timeout}s\n"
|
| 572 |
+
)
|
| 573 |
+
sys.stderr.flush()
|
| 574 |
+
|
| 575 |
+
step = 1
|
| 576 |
+
full_text, truncated = _generate(
|
| 577 |
+
base_prompt,
|
| 578 |
+
include_subtitle_text=True,
|
| 579 |
+
phase_label=f"Gemini thinking (step {step})",
|
| 580 |
+
base_lines=0,
|
| 581 |
+
)
|
| 582 |
+
if require_end_marker and END_MARKER in full_text:
|
| 583 |
+
return full_text
|
| 584 |
+
|
| 585 |
+
# If the model hit output limits, ask it to continue the same document.
|
| 586 |
+
rounds = 1
|
| 587 |
+
while truncated and rounds < max_rounds:
|
| 588 |
+
tail = full_text[-max(0, int(continuation_tail_chars)) :]
|
| 589 |
+
continuation_prompt = (
|
| 590 |
+
base_prompt
|
| 591 |
+
+ "\n\nYou already started writing the Markdown document. "
|
| 592 |
+
+ "However, your previous output was cut off due to length limits. "
|
| 593 |
+
+ "Remember all the original instructions. "
|
| 594 |
+
+ "Continue it from exactly where it stopped. Do NOT repeat content. "
|
| 595 |
+
+ "Maintain the same structure and formatting. "
|
| 596 |
+
+ ("Finish by writing the final line exactly equal to: " + END_MARKER + "\n" if require_end_marker else "")
|
| 597 |
+
+ "Here is the last part you wrote (for alignment):\n\n"
|
| 598 |
+
+ "```\n"
|
| 599 |
+
+ tail
|
| 600 |
+
+ "\n```\n\n"
|
| 601 |
+
+ "Continue now:\n"
|
| 602 |
+
)
|
| 603 |
+
# Continuations include the subtitle text to keep the model grounded and
|
| 604 |
+
# prevent the second half from drifting into generic filler.
|
| 605 |
+
step += 1
|
| 606 |
+
base_lines = full_text.count("\n") + (1 if full_text else 0)
|
| 607 |
+
next_text, truncated = _generate(
|
| 608 |
+
continuation_prompt,
|
| 609 |
+
include_subtitle_text=True,
|
| 610 |
+
phase_label=f"Gemini thinking (step {step})",
|
| 611 |
+
base_lines=base_lines,
|
| 612 |
+
)
|
| 613 |
+
|
| 614 |
+
# Best-effort de-duplication: if the continuation overlaps with the tail, trim it.
|
| 615 |
+
overlap_trimmed = next_text
|
| 616 |
+
if tail and next_text:
|
| 617 |
+
probe = tail[-200:]
|
| 618 |
+
pos = next_text.find(probe)
|
| 619 |
+
if pos != -1:
|
| 620 |
+
overlap_trimmed = next_text[pos + len(probe) :].lstrip()
|
| 621 |
+
|
| 622 |
+
if overlap_trimmed:
|
| 623 |
+
# Do not force a newline between chunks; it can break mid-line continuity.
|
| 624 |
+
left = full_text
|
| 625 |
+
right = overlap_trimmed.strip()
|
| 626 |
+
separator = ""
|
| 627 |
+
if left and right and left[-1].isalnum() and right[0].isalnum():
|
| 628 |
+
separator = " "
|
| 629 |
+
full_text = (left.rstrip() + separator + right).strip()
|
| 630 |
+
else:
|
| 631 |
+
# If we couldn't make progress, stop to avoid looping.
|
| 632 |
+
break
|
| 633 |
+
|
| 634 |
+
if require_end_marker and END_MARKER in full_text:
|
| 635 |
+
return full_text
|
| 636 |
+
|
| 637 |
+
rounds += 1
|
| 638 |
+
|
| 639 |
+
if require_end_marker and END_MARKER not in full_text:
|
| 640 |
+
raise RuntimeError(
|
| 641 |
+
"Gemini TLTW output was not finalized (missing END marker). "
|
| 642 |
+
"Try increasing max_rounds and/or max_output_tokens."
|
| 643 |
+
)
|
| 644 |
+
|
| 645 |
+
return full_text
|
| 646 |
+
|
| 647 |
+
|
| 648 |
+
def _strip_end_marker(text: str) -> str:
|
| 649 |
+
"""Remove the internal END marker from the final output."""
|
| 650 |
+
|
| 651 |
+
marker = "<!-- END -->"
|
| 652 |
+
if marker not in text:
|
| 653 |
+
return text
|
| 654 |
+
|
| 655 |
+
lines = text.splitlines()
|
| 656 |
+
cleaned = [line for line in lines if line.strip() != marker]
|
| 657 |
+
return "\n".join(cleaned).strip()
|
| 658 |
+
|
| 659 |
+
|
| 660 |
+
def generate_tltw_summary(
|
| 661 |
+
config: GeminiSummaryConfig,
|
| 662 |
+
*,
|
| 663 |
+
request_func: Callable[..., str] = _send_tltw_request,
|
| 664 |
+
) -> str:
|
| 665 |
+
"""
|
| 666 |
+
Generate a structured TLTW summary from an SRT file using Gemini.
|
| 667 |
+
|
| 668 |
+
Returns the generated summary text after writing it to ``config.output_file``.
|
| 669 |
+
"""
|
| 670 |
+
|
| 671 |
+
api_keys = normalize_api_keys(config.api_keys)
|
| 672 |
+
if not api_keys:
|
| 673 |
+
raise ValueError("Gemini API key is required for TLTW summaries. Provide --gemini_api_key.")
|
| 674 |
+
|
| 675 |
+
if not config.subtitle_file.exists():
|
| 676 |
+
raise FileNotFoundError(f"Subtitle file not found: {config.subtitle_file}")
|
| 677 |
+
|
| 678 |
+
subtitle_text = _load_srt_as_text(config.subtitle_file, config.truncate_chars)
|
| 679 |
+
duration_seconds = _estimate_srt_duration_seconds(config.subtitle_file)
|
| 680 |
+
# 10 items per 60 minutes, scaled by fractional hours (round up).
|
| 681 |
+
# Keep a floor of 10 for short videos to avoid undercoverage.
|
| 682 |
+
# Allow down to 40% smaller via min_key_points.
|
| 683 |
+
if duration_seconds <= 0:
|
| 684 |
+
max_key_points = 10
|
| 685 |
+
else:
|
| 686 |
+
hours = float(duration_seconds) / 3600.0
|
| 687 |
+
max_key_points = int(math.ceil(hours * 10.0))
|
| 688 |
+
max_key_points = max(10, max_key_points)
|
| 689 |
+
min_key_points = max(1, int(math.floor(max_key_points * 0.6)))
|
| 690 |
+
|
| 691 |
+
final_prompt_builder = lambda lang: _build_tltw_prompt_with_limits(
|
| 692 |
+
language=lang,
|
| 693 |
+
max_key_points=max_key_points,
|
| 694 |
+
min_key_points=min_key_points,
|
| 695 |
+
)
|
| 696 |
+
|
| 697 |
+
def _split_into_chunks(text: str, limit: int | None) -> list[str]:
|
| 698 |
+
if limit is None or limit <= 0 or len(text) <= limit:
|
| 699 |
+
return [text]
|
| 700 |
+
|
| 701 |
+
chunks = []
|
| 702 |
+
start = 0
|
| 703 |
+
while start < len(text):
|
| 704 |
+
end = min(len(text), start + limit)
|
| 705 |
+
chunks.append(text[start:end])
|
| 706 |
+
start = end
|
| 707 |
+
return chunks
|
| 708 |
+
|
| 709 |
+
chunks = _split_into_chunks(subtitle_text, config.chunk_chars)
|
| 710 |
+
|
| 711 |
+
def _run_request(payload_text: str, prompt_builder: Callable[[str], str]) -> str:
|
| 712 |
+
last_error: Exception | None = None
|
| 713 |
+
for key in api_keys:
|
| 714 |
+
try:
|
| 715 |
+
try:
|
| 716 |
+
return request_func(
|
| 717 |
+
api_key=key,
|
| 718 |
+
subtitle_text=payload_text,
|
| 719 |
+
language=config.language,
|
| 720 |
+
model=config.model,
|
| 721 |
+
max_output_tokens=config.max_output_tokens,
|
| 722 |
+
request_timeout=config.request_timeout,
|
| 723 |
+
prompt_builder=prompt_builder,
|
| 724 |
+
temperature=config.temperature,
|
| 725 |
+
top_p=config.top_p,
|
| 726 |
+
top_k=config.top_k,
|
| 727 |
+
max_rounds=config.max_rounds,
|
| 728 |
+
continuation_tail_chars=config.continuation_tail_chars,
|
| 729 |
+
show_progress=config.show_progress,
|
| 730 |
+
progress_update_interval=config.progress_update_interval,
|
| 731 |
+
progress_preview_chars=config.progress_preview_chars,
|
| 732 |
+
stream_output=config.stream_output,
|
| 733 |
+
)
|
| 734 |
+
except TypeError:
|
| 735 |
+
# Backward-compat for custom request_func implementations.
|
| 736 |
+
return request_func(
|
| 737 |
+
api_key=key,
|
| 738 |
+
subtitle_text=payload_text,
|
| 739 |
+
language=config.language,
|
| 740 |
+
model=config.model,
|
| 741 |
+
max_output_tokens=config.max_output_tokens,
|
| 742 |
+
request_timeout=config.request_timeout,
|
| 743 |
+
prompt_builder=prompt_builder,
|
| 744 |
+
temperature=config.temperature,
|
| 745 |
+
top_p=config.top_p,
|
| 746 |
+
top_k=config.top_k,
|
| 747 |
+
max_rounds=config.max_rounds,
|
| 748 |
+
continuation_tail_chars=config.continuation_tail_chars,
|
| 749 |
+
)
|
| 750 |
+
except Exception as exc: # noqa: BLE001
|
| 751 |
+
if config.show_progress:
|
| 752 |
+
sys.stderr.write(
|
| 753 |
+
f"Request failed with key={_mask_api_key(key)}: {exc}\n"
|
| 754 |
+
)
|
| 755 |
+
sys.stderr.flush()
|
| 756 |
+
last_error = exc
|
| 757 |
+
continue
|
| 758 |
+
raise RuntimeError("Gemini TLTW summary failed with all provided API keys.") from last_error
|
| 759 |
+
|
| 760 |
+
if len(chunks) == 1:
|
| 761 |
+
summary = _run_request(chunks[0], final_prompt_builder)
|
| 762 |
+
summary = _strip_end_marker(summary)
|
| 763 |
+
config.output_file.parent.mkdir(parents=True, exist_ok=True)
|
| 764 |
+
config.output_file.write_text(summary, encoding="utf-8")
|
| 765 |
+
return summary
|
| 766 |
+
|
| 767 |
+
# Multi-chunk: summarize each chunk, then synthesize
|
| 768 |
+
chunk_summaries: list[str] = []
|
| 769 |
+
total = len(chunks)
|
| 770 |
+
for idx, chunk in enumerate(chunks, start=1):
|
| 771 |
+
chunk_prompt = lambda lang, i=idx, t=total: _build_chunk_prompt(lang, i, t)
|
| 772 |
+
summary_chunk = _run_request(chunk, chunk_prompt)
|
| 773 |
+
chunk_summaries.append(f"### Chunk {idx}/{total}\n{summary_chunk}")
|
| 774 |
+
|
| 775 |
+
final_text = "\n\n".join(chunk_summaries)
|
| 776 |
+
|
| 777 |
+
final_tokens = config.final_max_output_tokens or config.max_output_tokens
|
| 778 |
+
|
| 779 |
+
# Final synthesis reuses request_func but with aggregated summaries
|
| 780 |
+
last_error: Exception | None = None
|
| 781 |
+
for key in api_keys:
|
| 782 |
+
try:
|
| 783 |
+
try:
|
| 784 |
+
final_summary = request_func(
|
| 785 |
+
api_key=key,
|
| 786 |
+
subtitle_text=f"Summaries of all chunks follow:\n\n{final_text}",
|
| 787 |
+
language=config.language,
|
| 788 |
+
model=config.model,
|
| 789 |
+
max_output_tokens=final_tokens,
|
| 790 |
+
request_timeout=config.request_timeout,
|
| 791 |
+
prompt_builder=final_prompt_builder,
|
| 792 |
+
temperature=config.temperature,
|
| 793 |
+
top_p=config.top_p,
|
| 794 |
+
top_k=config.top_k,
|
| 795 |
+
max_rounds=config.max_rounds,
|
| 796 |
+
continuation_tail_chars=config.continuation_tail_chars,
|
| 797 |
+
show_progress=config.show_progress,
|
| 798 |
+
progress_update_interval=config.progress_update_interval,
|
| 799 |
+
progress_preview_chars=config.progress_preview_chars,
|
| 800 |
+
stream_output=config.stream_output,
|
| 801 |
+
)
|
| 802 |
+
except TypeError:
|
| 803 |
+
final_summary = request_func(
|
| 804 |
+
api_key=key,
|
| 805 |
+
subtitle_text=f"Summaries of all chunks follow:\n\n{final_text}",
|
| 806 |
+
language=config.language,
|
| 807 |
+
model=config.model,
|
| 808 |
+
max_output_tokens=final_tokens,
|
| 809 |
+
request_timeout=config.request_timeout,
|
| 810 |
+
prompt_builder=final_prompt_builder,
|
| 811 |
+
temperature=config.temperature,
|
| 812 |
+
top_p=config.top_p,
|
| 813 |
+
top_k=config.top_k,
|
| 814 |
+
max_rounds=config.max_rounds,
|
| 815 |
+
continuation_tail_chars=config.continuation_tail_chars,
|
| 816 |
+
)
|
| 817 |
+
final_summary = _strip_end_marker(final_summary)
|
| 818 |
+
config.output_file.parent.mkdir(parents=True, exist_ok=True)
|
| 819 |
+
config.output_file.write_text(final_summary, encoding="utf-8")
|
| 820 |
+
return final_summary
|
| 821 |
+
except Exception as exc: # noqa: BLE001
|
| 822 |
+
if config.show_progress:
|
| 823 |
+
sys.stderr.write(
|
| 824 |
+
f"Final synthesis failed with key={_mask_api_key(key)}: {exc}\n"
|
| 825 |
+
)
|
| 826 |
+
sys.stderr.flush()
|
| 827 |
+
last_error = exc
|
| 828 |
+
continue
|
| 829 |
+
|
| 830 |
+
raise RuntimeError("Gemini TLTW summary failed during final synthesis with all provided API keys.") from last_error
|
| 831 |
+
|
| 832 |
+
|
| 833 |
+
def generate_tltw(
|
| 834 |
+
config: GeminiSummaryConfig,
|
| 835 |
+
*,
|
| 836 |
+
request_func: Callable[..., str] = _send_tltw_request,
|
| 837 |
+
) -> str:
|
| 838 |
+
"""Backward/short alias for generating a TLTW summary."""
|
| 839 |
+
|
| 840 |
+
return generate_tltw_summary(config, request_func=request_func)
|
install.bat
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
@echo off
|
| 2 |
+
echo Installing dependencies...
|
| 3 |
+
uv sync
|
| 4 |
+
echo Adding extra dependencies...
|
| 5 |
+
uv add gradio ffmpeg-progress-yield
|
| 6 |
+
echo.
|
| 7 |
+
echo Checking for FFmpeg...
|
| 8 |
+
bg-ffmpeg -h >nul 2>&1
|
| 9 |
+
if %errorlevel% neq 0 (
|
| 10 |
+
echo FFmpeg not found. Attempting to install via Winget...
|
| 11 |
+
winget install -e --id Gyan.FFmpeg
|
| 12 |
+
if %errorlevel% neq 0 (
|
| 13 |
+
echo.
|
| 14 |
+
echo WARNING: Failed to install FFmpeg automatically.
|
| 15 |
+
echo Please download it from https://ffmpeg.org/download.html and add to PATH.
|
| 16 |
+
echo.
|
| 17 |
+
) else (
|
| 18 |
+
echo FFmpeg installed successfully! Please restart your terminal/PC if needed.
|
| 19 |
+
)
|
| 20 |
+
) else (
|
| 21 |
+
echo FFmpeg is already installed.
|
| 22 |
+
)
|
| 23 |
+
pause
|
install_gpu.bat
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
@echo off
|
| 2 |
+
echo Uninstalling CPU pytorch...
|
| 3 |
+
uv pip uninstall torch torchvision torchaudio
|
| 4 |
+
echo Installing GPU Pytorch (CUDA 12.4)...
|
| 5 |
+
uv pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124
|
| 6 |
+
echo Done. You can now use 'start.bat'.
|
| 7 |
+
pause
|
legen-beta.ipynb
ADDED
|
@@ -0,0 +1,345 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "markdown",
|
| 5 |
+
"metadata": {},
|
| 6 |
+
"source": [
|
| 7 |
+
"# 🦁 LeGen **BETA**\n",
|
| 8 |
+
"\n",
|
| 9 |
+
"> ⚠️ **WARNING: BETA VERSION**\n",
|
| 10 |
+
">\n",
|
| 11 |
+
"> You are running the **BETA** version of LeGen.\n",
|
| 12 |
+
"> Features may be unstable or incomplete.\n",
|
| 13 |
+
"\n",
|
| 14 |
+
"---"
|
| 15 |
+
]
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"cell_type": "code",
|
| 19 |
+
"execution_count": null,
|
| 20 |
+
"metadata": {
|
| 21 |
+
"id": "G7E7eZwJ7F5c"
|
| 22 |
+
},
|
| 23 |
+
"outputs": [],
|
| 24 |
+
"source": [
|
| 25 |
+
"#@title *Prepare* { display-mode: \"form\", run: \"auto\" }\n",
|
| 26 |
+
"\n",
|
| 27 |
+
"#@markdown # **Prepare the environment**\n",
|
| 28 |
+
"#@markdown Execute to install necessary packages, modules, fonts\n",
|
| 29 |
+
"\n",
|
| 30 |
+
"#@markdown Mount Google Drive at /content/drive (your drive folder at /content/drive/MyDrive):\n",
|
| 31 |
+
"mount_gdrive = True #@param {type:\"boolean\"}\n",
|
| 32 |
+
"#@markdown Force mount again. Useful for bug cases:\n",
|
| 33 |
+
"force_remount = False #@param {type:\"boolean\"}\n",
|
| 34 |
+
"\n",
|
| 35 |
+
"# remove the old log\n",
|
| 36 |
+
"!rm -rf error_log.txt > /dev/null\n",
|
| 37 |
+
"\n",
|
| 38 |
+
"import os\n",
|
| 39 |
+
"import subprocess\n",
|
| 40 |
+
"import shutil\n",
|
| 41 |
+
"\n",
|
| 42 |
+
"try:\n",
|
| 43 |
+
" from google.colab import drive\n",
|
| 44 |
+
"except ImportError:\n",
|
| 45 |
+
" drive = None\n",
|
| 46 |
+
"\n",
|
| 47 |
+
"# mount Google Drive\n",
|
| 48 |
+
"if mount_gdrive:\n",
|
| 49 |
+
" if drive is None:\n",
|
| 50 |
+
" print(\"Mounting skipped: google.colab is unavailable in this environment.\")\n",
|
| 51 |
+
" else:\n",
|
| 52 |
+
" print(\"Mounting your Google Drive | Waiting user Allow Access | \", end='')\n",
|
| 53 |
+
" try:\n",
|
| 54 |
+
" drive.mount('/content/drive/', force_remount=force_remount)\n",
|
| 55 |
+
" except Exception as e:\n",
|
| 56 |
+
" print(f\"[✗]: {e}\")\n",
|
| 57 |
+
"\n",
|
| 58 |
+
"# install uv\n",
|
| 59 |
+
"print(\"Installing uv...\", end='')\n",
|
| 60 |
+
"if shutil.which('uv') is None:\n",
|
| 61 |
+
" with open('/content/error_log.txt', 'a') as f:\n",
|
| 62 |
+
" process = subprocess.Popen('pip3 install uv', shell=True, stderr=f)\n",
|
| 63 |
+
" return_code = process.wait()\n",
|
| 64 |
+
" print(\"[✔]\" if return_code == 0 else \"[✗]\")\n",
|
| 65 |
+
"else:\n",
|
| 66 |
+
" print(\"[✔] (already installed)\")\n",
|
| 67 |
+
"\n",
|
| 68 |
+
"# install legen\n",
|
| 69 |
+
"print(\"Installing LeGen...\", end='')\n",
|
| 70 |
+
"repo_url = \"https://github.com/matheusbach/legen.git\"\n",
|
| 71 |
+
"with open('/content/error_log.txt', 'a') as f:\n",
|
| 72 |
+
" process = subprocess.Popen(f'uv tool install --force git+{repo_url}', shell=True, stderr=f)\n",
|
| 73 |
+
" return_code = process.wait()\n",
|
| 74 |
+
" print(\"[✔]\" if return_code == 0 else \"[✗]\")\n",
|
| 75 |
+
"\n",
|
| 76 |
+
"# install libcudnn8\n",
|
| 77 |
+
"print(\"Install libcudnn8...\", end='')\n",
|
| 78 |
+
"with open('/content/error_log.txt', 'a') as f:\n",
|
| 79 |
+
" process = subprocess.Popen('sudo apt install -y libcudnn8', shell=True, stderr=f)\n",
|
| 80 |
+
" return_code = process.wait()\n",
|
| 81 |
+
" print(\"[✔]\" if return_code == 0 else \"[✗]\")\n",
|
| 82 |
+
"\n",
|
| 83 |
+
"# install ffmpeg and xvfb\n",
|
| 84 |
+
"print(\"Installing FFmpeg and xvfb...\", end='')\n",
|
| 85 |
+
"with open('/content/error_log.txt', 'a') as f:\n",
|
| 86 |
+
" process = subprocess.Popen('apt update -y ; apt install ffmpeg xvfb -y', shell=True, stderr=f)\n",
|
| 87 |
+
" return_code = process.wait()\n",
|
| 88 |
+
" print(\"[✔]\" if return_code == 0 else \"[✗]\")\n",
|
| 89 |
+
"\n",
|
| 90 |
+
"# install pip requirements.txt updating\n",
|
| 91 |
+
"print(\"Installing fonts...\", end='')\n",
|
| 92 |
+
"with open('/content/error_log.txt', 'a') as f:\n",
|
| 93 |
+
" process = subprocess.Popen('echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | debconf-set-selections && apt install -y ttf-mscorefonts-installer && fc-cache -f -v', shell=True, stderr=f)\n",
|
| 94 |
+
" return_code = process.wait()\n",
|
| 95 |
+
" print(\"[✔]\" if return_code == 0 else \"[✗]\")\n",
|
| 96 |
+
"\n",
|
| 97 |
+
"# create a virtual display\n",
|
| 98 |
+
"os.system('Xvfb :1 -screen 0 2560x1440x8 &') # create virtual display with size 1600x1200 and 8 bit color. Color can be changed to 24, 16 or 8\n",
|
| 99 |
+
"os.environ['DISPLAY'] = ':1.0' # tell X clients to use our virtual DISPLAY :1.0.\n",
|
| 100 |
+
"\n",
|
| 101 |
+
"print(\"\\nPreparation tasks done.\")"
|
| 102 |
+
]
|
| 103 |
+
},
|
| 104 |
+
{
|
| 105 |
+
"cell_type": "code",
|
| 106 |
+
"execution_count": null,
|
| 107 |
+
"metadata": {
|
| 108 |
+
"id": "Bz77fHta9JxG"
|
| 109 |
+
},
|
| 110 |
+
"outputs": [],
|
| 111 |
+
"source": [
|
| 112 |
+
"#@title *Configure* { display-mode: \"form\", run: \"auto\" }\n",
|
| 113 |
+
"#@markdown # **Define Software Settings**\n",
|
| 114 |
+
"#@markdown ---\n",
|
| 115 |
+
"#@markdown ## General Options\n",
|
| 116 |
+
"#@markdown Set where your files are located (your Drive is the base /content/drive/MyDrive):\n",
|
| 117 |
+
"import os\n",
|
| 118 |
+
"from urllib.parse import urlparse\n",
|
| 119 |
+
"\n",
|
| 120 |
+
"overwrite_existing = False #@param {type:\"boolean\"}\n",
|
| 121 |
+
"normalize = False #@param {type:\"boolean\"}\n",
|
| 122 |
+
"generate_srt = True #@param {type:\"boolean\"}\n",
|
| 123 |
+
"generate_txt = False #@param {type:\"boolean\"}\n",
|
| 124 |
+
"generate_embed_softsub = True #@param {type:\"boolean\"}\n",
|
| 125 |
+
"generate_hardsub = True #@param {type:\"boolean\"}\n",
|
| 126 |
+
"copy_extra_files = True #@param {type:\"boolean\"}\n",
|
| 127 |
+
"#@markdown Process existing input subtitle files (.srt) found in the input folder (translate/TLTW):\n",
|
| 128 |
+
"process_input_subs = False #@param {type:\"boolean\"}\n",
|
| 129 |
+
"\n",
|
| 130 |
+
"input_path = \"/content/drive/MyDrive/LeGen/media\" #@param {type:\"string\"}\n",
|
| 131 |
+
"download_path = \"/content/drive/MyDrive/LeGen/downloads\" #@param {type:\"string\"} # yt-dlp stores media here with subtitles embedded in the MP4\n",
|
| 132 |
+
"output_softsubs_path = \"/content/drive/MyDrive/LeGen/softsubs\" #@param {type:\"string\"}\n",
|
| 133 |
+
"output_hardsubs_path = \"/content/drive/MyDrive/LeGen/hardsubs\" #@param {type:\"string\"}\n",
|
| 134 |
+
"\n",
|
| 135 |
+
"def _looks_like_url(value: str) -> bool:\n",
|
| 136 |
+
" try:\n",
|
| 137 |
+
" parsed = urlparse(value)\n",
|
| 138 |
+
" return parsed.scheme in (\"http\", \"https\") and bool(parsed.netloc)\n",
|
| 139 |
+
" except Exception:\n",
|
| 140 |
+
" return False\n",
|
| 141 |
+
"\n",
|
| 142 |
+
"# Create directories helper\n",
|
| 143 |
+
"#@markdown Click the button below if you need to create the directories above:\n",
|
| 144 |
+
"if \"_legen_create_dirs\" not in globals():\n",
|
| 145 |
+
" def _legen_create_dirs():\n",
|
| 146 |
+
" if input_path and not _looks_like_url(input_path):\n",
|
| 147 |
+
" os.makedirs(input_path, exist_ok=True)\n",
|
| 148 |
+
" elif _looks_like_url(input_path):\n",
|
| 149 |
+
" print(\"Input is a URL. Skipping local directory creation for it.\")\n",
|
| 150 |
+
" if download_path:\n",
|
| 151 |
+
" os.makedirs(download_path, exist_ok=True)\n",
|
| 152 |
+
" if generate_embed_softsub:\n",
|
| 153 |
+
" os.makedirs(output_softsubs_path, exist_ok=True)\n",
|
| 154 |
+
" if generate_hardsub:\n",
|
| 155 |
+
" os.makedirs(output_hardsubs_path, exist_ok=True)\n",
|
| 156 |
+
" print(\"Directories ready.\")\n",
|
| 157 |
+
"try:\n",
|
| 158 |
+
" from google.colab import output as _colab_output\n",
|
| 159 |
+
" _colab_output.register_callback('legen_create_dirs', _legen_create_dirs)\n",
|
| 160 |
+
"except Exception:\n",
|
| 161 |
+
" pass\n",
|
| 162 |
+
"#@markdown\n",
|
| 163 |
+
"#@markdown <button onclick=\"google.colab.kernel.invokeFunction('legen_create_dirs', [], {});\">Create directories</button>\n",
|
| 164 |
+
"\n",
|
| 165 |
+
"#@markdown ---\n",
|
| 166 |
+
"#@markdown ## Transcription Settings:\n",
|
| 167 |
+
"transcription_engine = 'WhisperX' # @param [\"Whisper\", \"WhisperX\"]\n",
|
| 168 |
+
"transcription_device = 'auto' #@param [\"auto\", \"cpu\", \"cuda\"]\n",
|
| 169 |
+
"transcription_model = 'large-v3-turbo' #@param [\"tiny\", \"small\", \"medium\", \"large\", \"large-v1\", \"large-v2\", \"large-v3\", \"turbo\", \"large-v3-turbo\", \"distil-large-v2\", \"distil-medium.en\", \"distil-small.en\", \"distil-large-v3\"]\n",
|
| 170 |
+
"transcription_vad = 'Silero' # @param [\"Silero\", \"Pyannote\"]\n",
|
| 171 |
+
"compute_type = 'default' # @param [\"default\", \"int8\", \"int16\", \"float16\", \"float32\"]\n",
|
| 172 |
+
"batch_size = 12 #@param {type:\"number\"}\n",
|
| 173 |
+
"transcription_input_lang = 'auto detect' #@param [\"auto detect\", \"af\", \"am\", \"ar\", \"as\", \"az\", \"ba\", \"be\", \"bg\", \"bn\", \"bo\", \"br\", \"bs\", \"ca\", \"cs\", \"cy\", \"da\", \"de\", \"el\", \"en\", \"es\", \"et\", \"eu\", \"fa\", \"fi\", \"fo\", \"fr\", \"gl\", \"gu\", \"ha\", \"haw\", \"he\", \"hi\", \"hr\", \"ht\", \"hu\", \"hy\", \"id\", \"is\", \"it\", \"ja\", \"jw\", \"ka\", \"kk\", \"km\", \"kn\", \"ko\", \"la\", \"lb\", \"ln\", \"lo\", \"lt\", \"lv\", \"mg\", \"mi\", \"mk\", \"ml\", \"mn\", \"mr\", \"ms\", \"mt\", \"my\", \"ne\", \"nl\", \"nn\", \"no\", \"oc\", \"pa\", \"pl\", \"ps\", \"pt\", \"ro\", \"ru\", \"sa\", \"sd\", \"si\", \"sk\", \"sl\", \"sn\", \"so\", \"sq\", \"sr\", \"su\", \"sv\", \"sw\", \"ta\", \"te\", \"tg\", \"th\", \"tk\", \"tl\", \"tr\", \"tt\", \"uk\", \"ur\", \"uz\", \"vi\", \"yi\", \"yo\", \"zh\"]\n",
|
| 174 |
+
"\n",
|
| 175 |
+
"#@markdown ---\n",
|
| 176 |
+
"#@markdown ## Translation Settings:\n",
|
| 177 |
+
"#@markdown Set the destination language code. Set to same as original to skip translation\n",
|
| 178 |
+
"target_language_code = 'pt-BR' #@param [\"af\", \"sq\", \"am\", \"ar\", \"hy\", \"as\", \"ay\", \"az\", \"bm\", \"eu\", \"be\", \"bn\", \"bho\", \"bs\", \"bg\", \"ca\", \"ceb\", \"zh-CN\", \"zh-TW\", \"co\", \"hr\", \"cs\", \"da\", \"dv\", \"doi\", \"nl\", \"en\", \"eo\", \"et\", \"ee\", \"fil\", \"fi\", \"fr\", \"fy\", \"gl\", \"ka\", \"de\", \"el\", \"gn\", \"gu\", \"ht\", \"ha\", \"haw\", \"he\", \"hi\", \"hmn\", \"hu\", \"is\", \"ig\", \"ilo\", \"id\", \"ga\", \"it\", \"ja\", \"jv\", \"kn\", \"kk\", \"km\", \"rw\", \"gom\", \"ko\", \"kri\", \"ku\", \"ckb\", \"ky\", \"lo\", \"la\", \"lv\", \"ln\", \"lt\", \"lg\", \"lb\", \"mk\", \"mai\", \"mg\", \"ms\", \"ml\", \"mt\", \"mi\", \"mr\", \"mni-Mtei\", \"lus\", \"mn\", \"my\", \"ne\", \"no\", \"ny\", \"or\", \"om\", \"ps\", \"fa\", \"pl\", \"pt-BR\", \"pt-PT\", \"pa\", \"qu\", \"ro\", \"ru\", \"sm\", \"sa\", \"gd\", \"nso\", \"sr\", \"st\", \"sn\", \"sd\", \"si\", \"sk\", \"sl\", \"so\", \"es\", \"su\", \"sw\", \"sv\", \"tl\", \"tg\", \"ta\", \"tt\", \"te\", \"th\", \"ti\", \"ts\", \"tr\", \"tk\", \"ak\", \"uk\", \"ur\", \"ug\", \"uz\", \"vi\", \"cy\", \"xh\", \"yi\", \"yo\", \"zu\"]\n",
|
| 179 |
+
"#@markdown Select translation engine:\n",
|
| 180 |
+
"translate_engine = 'google' #@param [\"google\", \"gemini\"]\n",
|
| 181 |
+
"#@markdown If using Gemini (translation) and/or TLTW (summaries), paste your API keys here (separate with commas or new lines):\n",
|
| 182 |
+
"gemini_api_keys = '' #@param {type:\"string\"}\n",
|
| 183 |
+
"\n",
|
| 184 |
+
"#@markdown ---\n",
|
| 185 |
+
"#@markdown ## TLTW (Too Long To Watch)\n",
|
| 186 |
+
"#@markdown Generate a structured summary (.md) from subtitles (uses translated subtitles if generated):\n",
|
| 187 |
+
"enable_tltw = False #@param {type:\"boolean\"}\n",
|
| 188 |
+
"#@markdown Optional: output folder for TLTW summaries (leave empty to use softsubs folder):\n",
|
| 189 |
+
"tltw_output_path = \"\" #@param {type:\"string\"}\n",
|
| 190 |
+
"\n",
|
| 191 |
+
"#@markdown ---\n",
|
| 192 |
+
"#@markdown ## Video Settings:\n",
|
| 193 |
+
"codec_video = \"h264\" #@param [\"h264\", \"hevc\", \"mpeg4\"]\n",
|
| 194 |
+
"video_hardware_api = \"auto\" #@param [\"auto\", \"none\", \"nvenc\", \"vaapi\", \"amf\", \"v4l2m2m\", \"qsv\", \"videotoolbox\", \"cuvid\"]\n",
|
| 195 |
+
"codec_audio = \"aac\" #@param [\"aac\", \"libopus\", \"libmp3lame\", \"pcm_s16le\"]\n",
|
| 196 |
+
"\n",
|
| 197 |
+
"def collect_legen_settings():\n",
|
| 198 |
+
" subtitle_formats = []\n",
|
| 199 |
+
" if generate_srt:\n",
|
| 200 |
+
" subtitle_formats.append(\"srt\")\n",
|
| 201 |
+
" if generate_txt:\n",
|
| 202 |
+
" subtitle_formats.append(\"txt\")\n",
|
| 203 |
+
" if not subtitle_formats:\n",
|
| 204 |
+
" subtitle_formats = [\"srt\"]\n",
|
| 205 |
+
" gemini_keys = [\n",
|
| 206 |
+
" key.strip()\n",
|
| 207 |
+
" for key in gemini_api_keys.replace(',', '\\n').splitlines()\n",
|
| 208 |
+
" if key.strip()\n",
|
| 209 |
+
" ]\n",
|
| 210 |
+
" return {\n",
|
| 211 |
+
" \"input_path\": input_path,\n",
|
| 212 |
+
" \"download_path\": download_path,\n",
|
| 213 |
+
" \"output_softsubs_path\": output_softsubs_path,\n",
|
| 214 |
+
" \"output_hardsubs_path\": output_hardsubs_path,\n",
|
| 215 |
+
" \"overwrite_existing\": overwrite_existing,\n",
|
| 216 |
+
" \"normalize\": normalize,\n",
|
| 217 |
+
" \"copy_extra_files\": copy_extra_files,\n",
|
| 218 |
+
" \"generate_srt\": generate_srt,\n",
|
| 219 |
+
" \"generate_txt\": generate_txt,\n",
|
| 220 |
+
" \"generate_embed_softsub\": generate_embed_softsub,\n",
|
| 221 |
+
" \"generate_hardsub\": generate_hardsub,\n",
|
| 222 |
+
" \"process_input_subs\": process_input_subs,\n",
|
| 223 |
+
" \"transcription_engine\": transcription_engine,\n",
|
| 224 |
+
" \"transcription_device\": transcription_device,\n",
|
| 225 |
+
" \"transcription_model\": transcription_model,\n",
|
| 226 |
+
" \"transcription_vad\": transcription_vad,\n",
|
| 227 |
+
" \"compute_type\": compute_type,\n",
|
| 228 |
+
" \"batch_size\": batch_size if transcription_engine == 'WhisperX' else None,\n",
|
| 229 |
+
" \"transcription_input_lang\": transcription_input_lang,\n",
|
| 230 |
+
" \"target_language_code\": target_language_code,\n",
|
| 231 |
+
" \"translate_engine\": translate_engine,\n",
|
| 232 |
+
" \"gemini_api_keys\": gemini_keys,\n",
|
| 233 |
+
" \"enable_tltw\": enable_tltw,\n",
|
| 234 |
+
" \"tltw_output_path\": tltw_output_path,\n",
|
| 235 |
+
" \"codec_video\": codec_video,\n",
|
| 236 |
+
" \"video_hardware_api\": video_hardware_api,\n",
|
| 237 |
+
" \"codec_audio\": codec_audio,\n",
|
| 238 |
+
" \"subtitle_formats\": subtitle_formats,\n",
|
| 239 |
+
" \"input_is_url\": _looks_like_url(input_path),\n",
|
| 240 |
+
" }\n"
|
| 241 |
+
]
|
| 242 |
+
},
|
| 243 |
+
{
|
| 244 |
+
"cell_type": "code",
|
| 245 |
+
"execution_count": null,
|
| 246 |
+
"metadata": {
|
| 247 |
+
"id": "n4KSJ9BL7JOs"
|
| 248 |
+
},
|
| 249 |
+
"outputs": [],
|
| 250 |
+
"source": [
|
| 251 |
+
"#@title *Run* { display-mode: \"form\" }\n",
|
| 252 |
+
"#@markdown # **Run LeGen.py**\n",
|
| 253 |
+
"\n",
|
| 254 |
+
"if \"collect_legen_settings\" not in globals():\n",
|
| 255 |
+
" raise RuntimeError(\"Run the Configure cell before executing this step.\")\n",
|
| 256 |
+
"\n",
|
| 257 |
+
"print(\"Starting LeGen...\")\n",
|
| 258 |
+
"import os\n",
|
| 259 |
+
"import shlex\n",
|
| 260 |
+
"import torch\n",
|
| 261 |
+
"\n",
|
| 262 |
+
"settings = collect_legen_settings()\n",
|
| 263 |
+
"settings[\"transcription_vad\"] = settings[\"transcription_vad\"].lower()\n",
|
| 264 |
+
"\n",
|
| 265 |
+
"if settings.get(\"enable_tltw\") and not settings.get(\"gemini_api_keys\"):\n",
|
| 266 |
+
" raise ValueError(\"TLTW requires at least one Gemini API key. Paste it in the Configure cell.\")\n",
|
| 267 |
+
"\n",
|
| 268 |
+
"if '_looks_like_url' not in globals():\n",
|
| 269 |
+
" from urllib.parse import urlparse\n",
|
| 270 |
+
" def _looks_like_url(value: str) -> bool:\n",
|
| 271 |
+
" try:\n",
|
| 272 |
+
" parsed = urlparse(value)\n",
|
| 273 |
+
" return parsed.scheme in (\"http\", \"https\") and bool(parsed.netloc)\n",
|
| 274 |
+
" except Exception:\n",
|
| 275 |
+
" return False\n",
|
| 276 |
+
"\n",
|
| 277 |
+
"input_is_url = settings.get(\"input_is_url\", False)\n",
|
| 278 |
+
"if not input_is_url:\n",
|
| 279 |
+
" os.makedirs(settings[\"input_path\"], exist_ok=True)\n",
|
| 280 |
+
"else:\n",
|
| 281 |
+
" print(\"Input path is a URL. The downloader will fetch the media automatically.\")\n",
|
| 282 |
+
"if settings.get(\"download_path\"):\n",
|
| 283 |
+
" os.makedirs(settings[\"download_path\"], exist_ok=True)\n",
|
| 284 |
+
"if settings[\"generate_embed_softsub\"]:\n",
|
| 285 |
+
" os.makedirs(settings[\"output_softsubs_path\"], exist_ok=True)\n",
|
| 286 |
+
"if settings[\"generate_hardsub\"]:\n",
|
| 287 |
+
" os.makedirs(settings[\"output_hardsubs_path\"], exist_ok=True)\n",
|
| 288 |
+
"\n",
|
| 289 |
+
"subtitle_formats_value = ','.join(settings[\"subtitle_formats\"])\n",
|
| 290 |
+
"\n",
|
| 291 |
+
"# build query\n",
|
| 292 |
+
"query = f\" -i '{settings['input_path']}'\"\n",
|
| 293 |
+
"query += f\" --output_softsubs '{settings['output_softsubs_path']}'\"\n",
|
| 294 |
+
"query += f\" --output_hardsubs '{settings['output_hardsubs_path']}'\"\n",
|
| 295 |
+
"if settings.get(\"download_path\"):\n",
|
| 296 |
+
" query += f\" --output_downloads {shlex.quote(settings['download_path'])}\"\n",
|
| 297 |
+
"query += \" --process_input_subs\" if settings.get(\"process_input_subs\") else \"\"\n",
|
| 298 |
+
"query += \" --overwrite\" if settings[\"overwrite_existing\"] else \"\"\n",
|
| 299 |
+
"query += \" --norm\" if settings[\"normalize\"] else \"\"\n",
|
| 300 |
+
"query += \" --copy_files\" if not settings[\"copy_extra_files\"] else \"\"\n",
|
| 301 |
+
"query += f\" --subtitle_formats {shlex.quote(subtitle_formats_value)}\" if subtitle_formats_value else \"\"\n",
|
| 302 |
+
"query += \" --disable_softsubs\" if not settings[\"generate_embed_softsub\"] else \"\"\n",
|
| 303 |
+
"query += \" --disable_hardsubs\" if not settings[\"generate_hardsub\"] else \"\"\n",
|
| 304 |
+
"query += f\" -ts:e {settings['transcription_engine'].lower()}\"\n",
|
| 305 |
+
"query += f\" -ts:d {settings['transcription_device'].lower()}\"\n",
|
| 306 |
+
"query += f\" -ts:m {settings['transcription_model']}\"\n",
|
| 307 |
+
"query += f\" -ts:c {settings['compute_type']}\"\n",
|
| 308 |
+
"query += f\" -ts:v {settings['transcription_vad']}\"\n",
|
| 309 |
+
"query += f\" -ts:b {settings['batch_size']}\" if settings[\"batch_size\"] is not None else \"\"\n",
|
| 310 |
+
"query += f\" --input_lang {settings['transcription_input_lang']}\" if settings[\"transcription_input_lang\"] != \"auto detect\" else \"\"\n",
|
| 311 |
+
"query += f\" --translate {settings['target_language_code'].lower()}\"\n",
|
| 312 |
+
"query += f\" --translate_engine {settings['translate_engine']}\"\n",
|
| 313 |
+
"if settings.get(\"enable_tltw\"):\n",
|
| 314 |
+
" query += \" --tltw\"\n",
|
| 315 |
+
" if settings.get(\"tltw_output_path\"):\n",
|
| 316 |
+
" query += f\" --output_tltw {shlex.quote(settings['tltw_output_path'])}\"\n",
|
| 317 |
+
"if settings.get(\"translate_engine\") == \"gemini\" or settings.get(\"enable_tltw\"):\n",
|
| 318 |
+
" for key in settings[\"gemini_api_keys\"]:\n",
|
| 319 |
+
" query += f\" --gemini_api_key {shlex.quote(key)}\"\n",
|
| 320 |
+
"query += f\" -c:v {settings['codec_video']}\" + (\"\" if settings['video_hardware_api'] == \"none\" else f\"_{settings['video_hardware_api']}\" if settings['video_hardware_api'] != \"auto\" else \"_nvenc\" if torch.cuda.is_available() else \"\")\n",
|
| 321 |
+
"query += f\" -c:a {settings['codec_audio']}\"\n",
|
| 322 |
+
"\n",
|
| 323 |
+
"# run python script\n",
|
| 324 |
+
"print(f\"command line: legen {query}\", end=\"\\n\\n\")\n",
|
| 325 |
+
"!/root/.local/bin/legen $query\n"
|
| 326 |
+
]
|
| 327 |
+
}
|
| 328 |
+
],
|
| 329 |
+
"metadata": {
|
| 330 |
+
"accelerator": "GPU",
|
| 331 |
+
"colab": {
|
| 332 |
+
"gpuType": "T4",
|
| 333 |
+
"provenance": []
|
| 334 |
+
},
|
| 335 |
+
"kernelspec": {
|
| 336 |
+
"display_name": "Python 3",
|
| 337 |
+
"name": "python3"
|
| 338 |
+
},
|
| 339 |
+
"language_info": {
|
| 340 |
+
"name": "python"
|
| 341 |
+
}
|
| 342 |
+
},
|
| 343 |
+
"nbformat": 4,
|
| 344 |
+
"nbformat_minor": 0
|
| 345 |
+
}
|
legen.ipynb
CHANGED
|
@@ -1,227 +1,337 @@
|
|
| 1 |
-
{
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
"
|
| 40 |
-
|
| 41 |
-
"
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
"
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
"
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
"
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
"
|
| 204 |
-
"\n",
|
| 205 |
-
"
|
| 206 |
-
"
|
| 207 |
-
"
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
"
|
| 213 |
-
"
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
"
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
"
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "code",
|
| 5 |
+
"execution_count": null,
|
| 6 |
+
"metadata": {
|
| 7 |
+
"id": "G7E7eZwJ7F5c"
|
| 8 |
+
},
|
| 9 |
+
"outputs": [],
|
| 10 |
+
"source": [
|
| 11 |
+
"#@title *Prepare* { display-mode: \"form\", run: \"auto\" }\n",
|
| 12 |
+
"\n",
|
| 13 |
+
"#@markdown # **Prepare the environment**\n",
|
| 14 |
+
"#@markdown Execute to install necessary packages, modules, fonts\n",
|
| 15 |
+
"\n",
|
| 16 |
+
"#@markdown Select LeGen version (leave empty for latest):\n",
|
| 17 |
+
"legen_version = \"latest\" #@param {type:\"string\"}\n",
|
| 18 |
+
"\n",
|
| 19 |
+
"#@markdown Mount Google Drive at /content/drive (your drive folder at /content/drive/MyDrive):\n",
|
| 20 |
+
"mount_gdrive = True #@param {type:\"boolean\"}\n",
|
| 21 |
+
"#@markdown Force mount again. Useful for bug cases:\n",
|
| 22 |
+
"force_remount = False #@param {type:\"boolean\"}\n",
|
| 23 |
+
"\n",
|
| 24 |
+
"# remove the old log\n",
|
| 25 |
+
"!rm -rf error_log.txt > /dev/null\n",
|
| 26 |
+
"\n",
|
| 27 |
+
"import os\n",
|
| 28 |
+
"import subprocess\n",
|
| 29 |
+
"import shutil\n",
|
| 30 |
+
"\n",
|
| 31 |
+
"try:\n",
|
| 32 |
+
" from google.colab import drive\n",
|
| 33 |
+
"except ImportError:\n",
|
| 34 |
+
" drive = None\n",
|
| 35 |
+
"\n",
|
| 36 |
+
"# mount Google Drive\n",
|
| 37 |
+
"if mount_gdrive:\n",
|
| 38 |
+
" if drive is None:\n",
|
| 39 |
+
" print(\"Mounting skipped: google.colab is unavailable in this environment.\")\n",
|
| 40 |
+
" else:\n",
|
| 41 |
+
" print(\"Mounting your Google Drive | Waiting user Allow Access | \", end='')\n",
|
| 42 |
+
" try:\n",
|
| 43 |
+
" drive.mount('/content/drive/', force_remount=force_remount)\n",
|
| 44 |
+
" except Exception as e:\n",
|
| 45 |
+
" print(f\"[✗]: {e}\")\n",
|
| 46 |
+
"\n",
|
| 47 |
+
"# install uv\n",
|
| 48 |
+
"print(\"Installing uv...\", end='')\n",
|
| 49 |
+
"if shutil.which('uv') is None:\n",
|
| 50 |
+
" with open('/content/error_log.txt', 'a') as f:\n",
|
| 51 |
+
" process = subprocess.Popen('pip3 install uv', shell=True, stderr=f)\n",
|
| 52 |
+
" return_code = process.wait()\n",
|
| 53 |
+
" print(\"[✔]\" if return_code == 0 else \"[✗]\")\n",
|
| 54 |
+
"else:\n",
|
| 55 |
+
" print(\"[✔] (already installed)\")\n",
|
| 56 |
+
"\n",
|
| 57 |
+
"# install legen\n",
|
| 58 |
+
"print(f\"Installing LeGen ({legen_version})...\", end='')\n",
|
| 59 |
+
"install_cmd = 'uv tool install legen'\n",
|
| 60 |
+
"if legen_version and legen_version != \"latest\":\n",
|
| 61 |
+
" install_cmd = f'uv tool install legen=={legen_version}'\n",
|
| 62 |
+
"\n",
|
| 63 |
+
"with open('/content/error_log.txt', 'a') as f:\n",
|
| 64 |
+
" process = subprocess.Popen(install_cmd, shell=True, stderr=f)\n",
|
| 65 |
+
" return_code = process.wait()\n",
|
| 66 |
+
" print(\"[✔]\" if return_code == 0 else \"[✗]\")\n",
|
| 67 |
+
"\n",
|
| 68 |
+
"# install libcudnn8\n",
|
| 69 |
+
"print(\"Install libcudnn8...\", end='')\n",
|
| 70 |
+
"with open('/content/error_log.txt', 'a') as f:\n",
|
| 71 |
+
" process = subprocess.Popen('sudo apt install -y libcudnn8', shell=True, stderr=f)\n",
|
| 72 |
+
" return_code = process.wait()\n",
|
| 73 |
+
" print(\"[✔]\" if return_code == 0 else \"[✗]\")\n",
|
| 74 |
+
"\n",
|
| 75 |
+
"# install ffmpeg and xvfb\n",
|
| 76 |
+
"print(\"Installing FFmpeg and xvfb...\", end='')\n",
|
| 77 |
+
"with open('/content/error_log.txt', 'a') as f:\n",
|
| 78 |
+
" process = subprocess.Popen('apt update -y ; apt install ffmpeg xvfb -y', shell=True, stderr=f)\n",
|
| 79 |
+
" return_code = process.wait()\n",
|
| 80 |
+
" print(\"[✔]\" if return_code == 0 else \"[✗]\")\n",
|
| 81 |
+
"\n",
|
| 82 |
+
"# install pip requirements.txt updating\n",
|
| 83 |
+
"print(\"Installing fonts...\", end='')\n",
|
| 84 |
+
"with open('/content/error_log.txt', 'a') as f:\n",
|
| 85 |
+
" process = subprocess.Popen('echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | debconf-set-selections && apt install -y ttf-mscorefonts-installer && fc-cache -f -v', shell=True, stderr=f)\n",
|
| 86 |
+
" return_code = process.wait()\n",
|
| 87 |
+
" print(\"[✔]\" if return_code == 0 else \"[✗]\")\n",
|
| 88 |
+
"\n",
|
| 89 |
+
"# create a virtual display\n",
|
| 90 |
+
"os.system('Xvfb :1 -screen 0 2560x1440x8 &') # create virtual display with size 1600x1200 and 8 bit color. Color can be changed to 24, 16 or 8\n",
|
| 91 |
+
"os.environ['DISPLAY'] = ':1.0' # tell X clients to use our virtual DISPLAY :1.0.\n",
|
| 92 |
+
"\n",
|
| 93 |
+
"print(\"\\nPreparation tasks done.\")"
|
| 94 |
+
]
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"cell_type": "code",
|
| 98 |
+
"execution_count": null,
|
| 99 |
+
"metadata": {
|
| 100 |
+
"id": "Bz77fHta9JxG"
|
| 101 |
+
},
|
| 102 |
+
"outputs": [],
|
| 103 |
+
"source": [
|
| 104 |
+
"#@title *Configure* { display-mode: \"form\", run: \"auto\" }\n",
|
| 105 |
+
"#@markdown # **Define Software Settings**\n",
|
| 106 |
+
"#@markdown ---\n",
|
| 107 |
+
"#@markdown ## General Options\n",
|
| 108 |
+
"#@markdown Set where your files are located (your Drive is the base /content/drive/MyDrive):\n",
|
| 109 |
+
"import os\n",
|
| 110 |
+
"from urllib.parse import urlparse\n",
|
| 111 |
+
"\n",
|
| 112 |
+
"overwrite_existing = False #@param {type:\"boolean\"}\n",
|
| 113 |
+
"normalize = False #@param {type:\"boolean\"}\n",
|
| 114 |
+
"generate_srt = True #@param {type:\"boolean\"}\n",
|
| 115 |
+
"generate_txt = False #@param {type:\"boolean\"}\n",
|
| 116 |
+
"generate_embed_softsub = True #@param {type:\"boolean\"}\n",
|
| 117 |
+
"generate_hardsub = True #@param {type:\"boolean\"}\n",
|
| 118 |
+
"copy_extra_files = True #@param {type:\"boolean\"}\n",
|
| 119 |
+
"#@markdown Process existing input subtitle files (.srt) found in the input folder (translate/TLTW):\n",
|
| 120 |
+
"process_input_subs = False #@param {type:\"boolean\"}\n",
|
| 121 |
+
"\n",
|
| 122 |
+
"input_path = \"/content/drive/MyDrive/LeGen/media\" #@param {type:\"string\"}\n",
|
| 123 |
+
"download_path = \"/content/drive/MyDrive/LeGen/downloads\" #@param {type:\"string\"} # yt-dlp stores media here with subtitles embedded in the MP4\n",
|
| 124 |
+
"output_softsubs_path = \"/content/drive/MyDrive/LeGen/softsubs\" #@param {type:\"string\"}\n",
|
| 125 |
+
"output_hardsubs_path = \"/content/drive/MyDrive/LeGen/hardsubs\" #@param {type:\"string\"}\n",
|
| 126 |
+
"\n",
|
| 127 |
+
"def _looks_like_url(value: str) -> bool:\n",
|
| 128 |
+
" try:\n",
|
| 129 |
+
" parsed = urlparse(value)\n",
|
| 130 |
+
" return parsed.scheme in (\"http\", \"https\") and bool(parsed.netloc)\n",
|
| 131 |
+
" except Exception:\n",
|
| 132 |
+
" return False\n",
|
| 133 |
+
"\n",
|
| 134 |
+
"# Create directories helper\n",
|
| 135 |
+
"#@markdown Click the button below if you need to create the directories above:\n",
|
| 136 |
+
"if \"_legen_create_dirs\" not in globals():\n",
|
| 137 |
+
" def _legen_create_dirs():\n",
|
| 138 |
+
" if input_path and not _looks_like_url(input_path):\n",
|
| 139 |
+
" os.makedirs(input_path, exist_ok=True)\n",
|
| 140 |
+
" elif _looks_like_url(input_path):\n",
|
| 141 |
+
" print(\"Input is a URL. Skipping local directory creation for it.\")\n",
|
| 142 |
+
" if download_path:\n",
|
| 143 |
+
" os.makedirs(download_path, exist_ok=True)\n",
|
| 144 |
+
" if generate_embed_softsub:\n",
|
| 145 |
+
" os.makedirs(output_softsubs_path, exist_ok=True)\n",
|
| 146 |
+
" if generate_hardsub:\n",
|
| 147 |
+
" os.makedirs(output_hardsubs_path, exist_ok=True)\n",
|
| 148 |
+
" print(\"Directories ready.\")\n",
|
| 149 |
+
"try:\n",
|
| 150 |
+
" from google.colab import output as _colab_output\n",
|
| 151 |
+
" _colab_output.register_callback('legen_create_dirs', _legen_create_dirs)\n",
|
| 152 |
+
"except Exception:\n",
|
| 153 |
+
" pass\n",
|
| 154 |
+
"#@markdown\n",
|
| 155 |
+
"#@markdown <button onclick=\"google.colab.kernel.invokeFunction('legen_create_dirs', [], {});\">Create directories</button>\n",
|
| 156 |
+
"\n",
|
| 157 |
+
"#@markdown ---\n",
|
| 158 |
+
"#@markdown ## Transcription Settings:\n",
|
| 159 |
+
"transcription_engine = 'WhisperX' # @param [\"Whisper\", \"WhisperX\"]\n",
|
| 160 |
+
"transcription_device = 'auto' #@param [\"auto\", \"cpu\", \"cuda\"]\n",
|
| 161 |
+
"transcription_model = 'large-v3-turbo' #@param [\"tiny\", \"small\", \"medium\", \"large\", \"large-v1\", \"large-v2\", \"large-v3\", \"turbo\", \"large-v3-turbo\", \"distil-large-v2\", \"distil-medium.en\", \"distil-small.en\", \"distil-large-v3\"]\n",
|
| 162 |
+
"transcription_vad = 'Silero' # @param [\"Silero\", \"Pyannote\"]\n",
|
| 163 |
+
"compute_type = 'default' # @param [\"default\", \"int8\", \"int16\", \"float16\", \"float32\"]\n",
|
| 164 |
+
"batch_size = 12 #@param {type:\"number\"}\n",
|
| 165 |
+
"transcription_input_lang = 'auto detect' #@param [\"auto detect\", \"af\", \"am\", \"ar\", \"as\", \"az\", \"ba\", \"be\", \"bg\", \"bn\", \"bo\", \"br\", \"bs\", \"ca\", \"cs\", \"cy\", \"da\", \"de\", \"el\", \"en\", \"es\", \"et\", \"eu\", \"fa\", \"fi\", \"fo\", \"fr\", \"gl\", \"gu\", \"ha\", \"haw\", \"he\", \"hi\", \"hr\", \"ht\", \"hu\", \"hy\", \"id\", \"is\", \"it\", \"ja\", \"jw\", \"ka\", \"kk\", \"km\", \"kn\", \"ko\", \"la\", \"lb\", \"ln\", \"lo\", \"lt\", \"lv\", \"mg\", \"mi\", \"mk\", \"ml\", \"mn\", \"mr\", \"ms\", \"mt\", \"my\", \"ne\", \"nl\", \"nn\", \"no\", \"oc\", \"pa\", \"pl\", \"ps\", \"pt\", \"ro\", \"ru\", \"sa\", \"sd\", \"si\", \"sk\", \"sl\", \"sn\", \"so\", \"sq\", \"sr\", \"su\", \"sv\", \"sw\", \"ta\", \"te\", \"tg\", \"th\", \"tk\", \"tl\", \"tr\", \"tt\", \"uk\", \"ur\", \"uz\", \"vi\", \"yi\", \"yo\", \"zh\"]\n",
|
| 166 |
+
"\n",
|
| 167 |
+
"#@markdown ---\n",
|
| 168 |
+
"#@markdown ## Translation Settings:\n",
|
| 169 |
+
"#@markdown Set the destination language code. Set to same as original to skip translation\n",
|
| 170 |
+
"target_language_code = 'pt-BR' #@param [\"af\", \"sq\", \"am\", \"ar\", \"hy\", \"as\", \"ay\", \"az\", \"bm\", \"eu\", \"be\", \"bn\", \"bho\", \"bs\", \"bg\", \"ca\", \"ceb\", \"zh-CN\", \"zh-TW\", \"co\", \"hr\", \"cs\", \"da\", \"dv\", \"doi\", \"nl\", \"en\", \"eo\", \"et\", \"ee\", \"fil\", \"fi\", \"fr\", \"fy\", \"gl\", \"ka\", \"de\", \"el\", \"gn\", \"gu\", \"ht\", \"ha\", \"haw\", \"he\", \"hi\", \"hmn\", \"hu\", \"is\", \"ig\", \"ilo\", \"id\", \"ga\", \"it\", \"ja\", \"jv\", \"kn\", \"kk\", \"km\", \"rw\", \"gom\", \"ko\", \"kri\", \"ku\", \"ckb\", \"ky\", \"lo\", \"la\", \"lv\", \"ln\", \"lt\", \"lg\", \"lb\", \"mk\", \"mai\", \"mg\", \"ms\", \"ml\", \"mt\", \"mi\", \"mr\", \"mni-Mtei\", \"lus\", \"mn\", \"my\", \"ne\", \"no\", \"ny\", \"or\", \"om\", \"ps\", \"fa\", \"pl\", \"pt-BR\", \"pt-PT\", \"pa\", \"qu\", \"ro\", \"ru\", \"sm\", \"sa\", \"gd\", \"nso\", \"sr\", \"st\", \"sn\", \"sd\", \"si\", \"sk\", \"sl\", \"so\", \"es\", \"su\", \"sw\", \"sv\", \"tl\", \"tg\", \"ta\", \"tt\", \"te\", \"th\", \"ti\", \"ts\", \"tr\", \"tk\", \"ak\", \"uk\", \"ur\", \"ug\", \"uz\", \"vi\", \"cy\", \"xh\", \"yi\", \"yo\", \"zu\"]\n",
|
| 171 |
+
"#@markdown Select translation engine:\n",
|
| 172 |
+
"translate_engine = 'google' #@param [\"google\", \"gemini\"]\n",
|
| 173 |
+
"#@markdown If using Gemini (translation) and/or TLTW (summaries), paste your API keys here (separate with commas or new lines):\n",
|
| 174 |
+
"gemini_api_keys = '' #@param {type:\"string\"}\n",
|
| 175 |
+
"\n",
|
| 176 |
+
"#@markdown ---\n",
|
| 177 |
+
"#@markdown ## TLTW (Too Long To Watch)\n",
|
| 178 |
+
"#@markdown Generate a structured summary (.md) from subtitles (uses translated subtitles if generated):\n",
|
| 179 |
+
"enable_tltw = False #@param {type:\"boolean\"}\n",
|
| 180 |
+
"#@markdown Optional: output folder for TLTW summaries (leave empty to use softsubs folder):\n",
|
| 181 |
+
"tltw_output_path = \"\" #@param {type:\"string\"}\n",
|
| 182 |
+
"\n",
|
| 183 |
+
"#@markdown ---\n",
|
| 184 |
+
"#@markdown ## Video Settings:\n",
|
| 185 |
+
"codec_video = \"h264\" #@param [\"h264\", \"hevc\", \"mpeg4\"]\n",
|
| 186 |
+
"video_hardware_api = \"auto\" #@param [\"auto\", \"none\", \"nvenc\", \"vaapi\", \"amf\", \"v4l2m2m\", \"qsv\", \"videotoolbox\", \"cuvid\"]\n",
|
| 187 |
+
"codec_audio = \"aac\" #@param [\"aac\", \"libopus\", \"libmp3lame\", \"pcm_s16le\"]\n",
|
| 188 |
+
"\n",
|
| 189 |
+
"def collect_legen_settings():\n",
|
| 190 |
+
" subtitle_formats = []\n",
|
| 191 |
+
" if generate_srt:\n",
|
| 192 |
+
" subtitle_formats.append(\"srt\")\n",
|
| 193 |
+
" if generate_txt:\n",
|
| 194 |
+
" subtitle_formats.append(\"txt\")\n",
|
| 195 |
+
" if not subtitle_formats:\n",
|
| 196 |
+
" subtitle_formats = [\"srt\"]\n",
|
| 197 |
+
" gemini_keys = [\n",
|
| 198 |
+
" key.strip()\n",
|
| 199 |
+
" for key in gemini_api_keys.replace(',', '\\n').splitlines()\n",
|
| 200 |
+
" if key.strip()\n",
|
| 201 |
+
" ]\n",
|
| 202 |
+
" return {\n",
|
| 203 |
+
" \"input_path\": input_path,\n",
|
| 204 |
+
" \"download_path\": download_path,\n",
|
| 205 |
+
" \"output_softsubs_path\": output_softsubs_path,\n",
|
| 206 |
+
" \"output_hardsubs_path\": output_hardsubs_path,\n",
|
| 207 |
+
" \"overwrite_existing\": overwrite_existing,\n",
|
| 208 |
+
" \"normalize\": normalize,\n",
|
| 209 |
+
" \"copy_extra_files\": copy_extra_files,\n",
|
| 210 |
+
" \"generate_srt\": generate_srt,\n",
|
| 211 |
+
" \"generate_txt\": generate_txt,\n",
|
| 212 |
+
" \"generate_embed_softsub\": generate_embed_softsub,\n",
|
| 213 |
+
" \"generate_hardsub\": generate_hardsub,\n",
|
| 214 |
+
" \"process_input_subs\": process_input_subs,\n",
|
| 215 |
+
" \"transcription_engine\": transcription_engine,\n",
|
| 216 |
+
" \"transcription_device\": transcription_device,\n",
|
| 217 |
+
" \"transcription_model\": transcription_model,\n",
|
| 218 |
+
" \"transcription_vad\": transcription_vad,\n",
|
| 219 |
+
" \"compute_type\": compute_type,\n",
|
| 220 |
+
" \"batch_size\": batch_size if transcription_engine == 'WhisperX' else None,\n",
|
| 221 |
+
" \"transcription_input_lang\": transcription_input_lang,\n",
|
| 222 |
+
" \"target_language_code\": target_language_code,\n",
|
| 223 |
+
" \"translate_engine\": translate_engine,\n",
|
| 224 |
+
" \"gemini_api_keys\": gemini_keys,\n",
|
| 225 |
+
" \"enable_tltw\": enable_tltw,\n",
|
| 226 |
+
" \"tltw_output_path\": tltw_output_path,\n",
|
| 227 |
+
" \"codec_video\": codec_video,\n",
|
| 228 |
+
" \"video_hardware_api\": video_hardware_api,\n",
|
| 229 |
+
" \"codec_audio\": codec_audio,\n",
|
| 230 |
+
" \"subtitle_formats\": subtitle_formats,\n",
|
| 231 |
+
" \"input_is_url\": _looks_like_url(input_path),\n",
|
| 232 |
+
" }\n"
|
| 233 |
+
]
|
| 234 |
+
},
|
| 235 |
+
{
|
| 236 |
+
"cell_type": "code",
|
| 237 |
+
"execution_count": null,
|
| 238 |
+
"metadata": {
|
| 239 |
+
"id": "n4KSJ9BL7JOs"
|
| 240 |
+
},
|
| 241 |
+
"outputs": [],
|
| 242 |
+
"source": [
|
| 243 |
+
"#@title *Run* { display-mode: \"form\" }\n",
|
| 244 |
+
"#@markdown # **Run LeGen.py**\n",
|
| 245 |
+
"\n",
|
| 246 |
+
"if \"collect_legen_settings\" not in globals():\n",
|
| 247 |
+
" raise RuntimeError(\"Run the Configure cell before executing this step.\")\n",
|
| 248 |
+
"\n",
|
| 249 |
+
"print(\"Starting LeGen...\")\n",
|
| 250 |
+
"import os\n",
|
| 251 |
+
"import shlex\n",
|
| 252 |
+
"import torch\n",
|
| 253 |
+
"\n",
|
| 254 |
+
"settings = collect_legen_settings()\n",
|
| 255 |
+
"settings[\"transcription_vad\"] = settings[\"transcription_vad\"].lower()\n",
|
| 256 |
+
"\n",
|
| 257 |
+
"if settings.get(\"enable_tltw\") and not settings.get(\"gemini_api_keys\"):\n",
|
| 258 |
+
" raise ValueError(\"TLTW requires at least one Gemini API key. Paste it in the Configure cell.\")\n",
|
| 259 |
+
"\n",
|
| 260 |
+
"if '_looks_like_url' not in globals():\n",
|
| 261 |
+
" from urllib.parse import urlparse\n",
|
| 262 |
+
" def _looks_like_url(value: str) -> bool:\n",
|
| 263 |
+
" try:\n",
|
| 264 |
+
" parsed = urlparse(value)\n",
|
| 265 |
+
" return parsed.scheme in (\"http\", \"https\") and bool(parsed.netloc)\n",
|
| 266 |
+
" except Exception:\n",
|
| 267 |
+
" return False\n",
|
| 268 |
+
"\n",
|
| 269 |
+
"input_is_url = settings.get(\"input_is_url\", False)\n",
|
| 270 |
+
"if not input_is_url:\n",
|
| 271 |
+
" os.makedirs(settings[\"input_path\"], exist_ok=True)\n",
|
| 272 |
+
"else:\n",
|
| 273 |
+
" print(\"Input path is a URL. The downloader will fetch the media automatically.\")\n",
|
| 274 |
+
"if settings.get(\"download_path\"):\n",
|
| 275 |
+
" os.makedirs(settings[\"download_path\"], exist_ok=True)\n",
|
| 276 |
+
"if settings[\"generate_embed_softsub\"]:\n",
|
| 277 |
+
" os.makedirs(settings[\"output_softsubs_path\"], exist_ok=True)\n",
|
| 278 |
+
"if settings[\"generate_hardsub\"]:\n",
|
| 279 |
+
" os.makedirs(settings[\"output_hardsubs_path\"], exist_ok=True)\n",
|
| 280 |
+
"\n",
|
| 281 |
+
"subtitle_formats_value = ','.join(settings[\"subtitle_formats\"])\n",
|
| 282 |
+
"\n",
|
| 283 |
+
"# build query\n",
|
| 284 |
+
"query = f\" -i '{settings['input_path']}'\"\n",
|
| 285 |
+
"query += f\" --output_softsubs '{settings['output_softsubs_path']}'\"\n",
|
| 286 |
+
"query += f\" --output_hardsubs '{settings['output_hardsubs_path']}'\"\n",
|
| 287 |
+
"if settings.get(\"download_path\"):\n",
|
| 288 |
+
" query += f\" --output_downloads {shlex.quote(settings['download_path'])}\"\n",
|
| 289 |
+
"query += \" --process_input_subs\" if settings.get(\"process_input_subs\") else \"\"\n",
|
| 290 |
+
"query += \" --overwrite\" if settings[\"overwrite_existing\"] else \"\"\n",
|
| 291 |
+
"query += \" --norm\" if settings[\"normalize\"] else \"\"\n",
|
| 292 |
+
"query += \" --copy_files\" if not settings[\"copy_extra_files\"] else \"\"\n",
|
| 293 |
+
"query += f\" --subtitle_formats {shlex.quote(subtitle_formats_value)}\" if subtitle_formats_value else \"\"\n",
|
| 294 |
+
"query += \" --disable_softsubs\" if not settings[\"generate_embed_softsub\"] else \"\"\n",
|
| 295 |
+
"query += \" --disable_hardsubs\" if not settings[\"generate_hardsub\"] else \"\"\n",
|
| 296 |
+
"query += f\" -ts:e {settings['transcription_engine'].lower()}\"\n",
|
| 297 |
+
"query += f\" -ts:d {settings['transcription_device'].lower()}\"\n",
|
| 298 |
+
"query += f\" -ts:m {settings['transcription_model']}\"\n",
|
| 299 |
+
"query += f\" -ts:c {settings['compute_type']}\"\n",
|
| 300 |
+
"query += f\" -ts:v {settings['transcription_vad']}\"\n",
|
| 301 |
+
"query += f\" -ts:b {settings['batch_size']}\" if settings[\"batch_size\"] is not None else \"\"\n",
|
| 302 |
+
"query += f\" --input_lang {settings['transcription_input_lang']}\" if settings[\"transcription_input_lang\"] != \"auto detect\" else \"\"\n",
|
| 303 |
+
"query += f\" --translate {settings['target_language_code'].lower()}\"\n",
|
| 304 |
+
"query += f\" --translate_engine {settings['translate_engine']}\"\n",
|
| 305 |
+
"if settings.get(\"enable_tltw\"):\n",
|
| 306 |
+
" query += \" --tltw\"\n",
|
| 307 |
+
" if settings.get(\"tltw_output_path\"):\n",
|
| 308 |
+
" query += f\" --output_tltw {shlex.quote(settings['tltw_output_path'])}\"\n",
|
| 309 |
+
"if settings.get(\"translate_engine\") == \"gemini\" or settings.get(\"enable_tltw\"):\n",
|
| 310 |
+
" for key in settings[\"gemini_api_keys\"]:\n",
|
| 311 |
+
" query += f\" --gemini_api_key {shlex.quote(key)}\"\n",
|
| 312 |
+
"query += f\" -c:v {settings['codec_video']}\" + (\"\" if settings['video_hardware_api'] == \"none\" else f\"_{settings['video_hardware_api']}\" if settings['video_hardware_api'] != \"auto\" else \"_nvenc\" if torch.cuda.is_available() else \"\")\n",
|
| 313 |
+
"query += f\" -c:a {settings['codec_audio']}\"\n",
|
| 314 |
+
"\n",
|
| 315 |
+
"# run python script\n",
|
| 316 |
+
"print(f\"command line: legen {query}\", end=\"\\n\\n\")\n",
|
| 317 |
+
"!/root/.local/bin/legen $query\n"
|
| 318 |
+
]
|
| 319 |
+
}
|
| 320 |
+
],
|
| 321 |
+
"metadata": {
|
| 322 |
+
"accelerator": "GPU",
|
| 323 |
+
"colab": {
|
| 324 |
+
"gpuType": "T4",
|
| 325 |
+
"provenance": []
|
| 326 |
+
},
|
| 327 |
+
"kernelspec": {
|
| 328 |
+
"display_name": "Python 3",
|
| 329 |
+
"name": "python3"
|
| 330 |
+
},
|
| 331 |
+
"language_info": {
|
| 332 |
+
"name": "python"
|
| 333 |
+
}
|
| 334 |
+
},
|
| 335 |
+
"nbformat": 4,
|
| 336 |
+
"nbformat_minor": 0
|
| 337 |
+
}
|
legen.py
CHANGED
|
@@ -1,280 +1,742 @@
|
|
| 1 |
-
import
|
| 2 |
-
|
| 3 |
-
import
|
| 4 |
-
import
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
import
|
| 9 |
-
import
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import os
|
| 5 |
+
import subprocess
|
| 6 |
+
import sys
|
| 7 |
+
import time
|
| 8 |
+
import warnings
|
| 9 |
+
import logging
|
| 10 |
+
|
| 11 |
+
# Filter warnings early
|
| 12 |
+
warnings.filterwarnings("ignore", category=SyntaxWarning, module=r"pyannote\.database.*")
|
| 13 |
+
warnings.filterwarnings("ignore", category=UserWarning, message="pkg_resources is deprecated as an API")
|
| 14 |
+
warnings.filterwarnings("ignore", category=UserWarning, message="torchaudio._backend.list_audio_backends has been deprecated",)
|
| 15 |
+
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
|
| 16 |
+
|
| 17 |
+
from inspect import currentframe, getframeinfo
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
from typing import Sequence
|
| 20 |
+
from urllib.parse import urlparse
|
| 21 |
+
|
| 22 |
+
import download_utils
|
| 23 |
+
import ffmpeg_utils
|
| 24 |
+
import file_utils
|
| 25 |
+
import subtitle_utils
|
| 26 |
+
import translate_utils
|
| 27 |
+
from gemini_utils import GeminiSummaryConfig, generate_tltw, normalize_api_keys
|
| 28 |
+
from utils import audio_extensions, check_other_extensions, split_lang_suffix, time_task, video_extensions
|
| 29 |
+
|
| 30 |
+
# Fix for matplotlib backend issue in some environments (e.g. Colab)
|
| 31 |
+
if os.environ.get("MPLBACKEND") == "module://matplotlib_inline.backend_inline":
|
| 32 |
+
os.environ.pop("MPLBACKEND")
|
| 33 |
+
|
| 34 |
+
VERSION = "0.20.0"
|
| 35 |
+
version = f"v{VERSION}"
|
| 36 |
+
__version__ = VERSION
|
| 37 |
+
__all__ = [
|
| 38 |
+
"VERSION",
|
| 39 |
+
"__version__",
|
| 40 |
+
"build_parser",
|
| 41 |
+
"looks_like_url",
|
| 42 |
+
"main",
|
| 43 |
+
"normalize_subtitle_formats",
|
| 44 |
+
]
|
| 45 |
+
|
| 46 |
+
SUPPORTED_SUBTITLE_FORMATS = {"srt", "txt"}
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def looks_like_url(value: str) -> bool:
|
| 50 |
+
if not value:
|
| 51 |
+
return False
|
| 52 |
+
parsed = urlparse(str(value))
|
| 53 |
+
return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def normalize_subtitle_formats(raw_value):
|
| 57 |
+
if isinstance(raw_value, (list, tuple)):
|
| 58 |
+
values = raw_value
|
| 59 |
+
else:
|
| 60 |
+
values = [raw_value]
|
| 61 |
+
|
| 62 |
+
formats = []
|
| 63 |
+
for value in values:
|
| 64 |
+
if value is None:
|
| 65 |
+
continue
|
| 66 |
+
parts = str(value).replace(",", " ").split()
|
| 67 |
+
for part in parts:
|
| 68 |
+
fmt = part.strip().lower()
|
| 69 |
+
if fmt:
|
| 70 |
+
formats.append(fmt)
|
| 71 |
+
|
| 72 |
+
if not formats:
|
| 73 |
+
formats = ["srt"]
|
| 74 |
+
|
| 75 |
+
# maintain user order while removing duplicates
|
| 76 |
+
seen = {}
|
| 77 |
+
for fmt in formats:
|
| 78 |
+
seen.setdefault(fmt, None)
|
| 79 |
+
|
| 80 |
+
return list(seen.keys())
|
| 81 |
+
|
| 82 |
+
# Terminal colors
|
| 83 |
+
default = "\033[1;0m"
|
| 84 |
+
gray = "\033[1;37m"
|
| 85 |
+
wblue = "\033[1;36m"
|
| 86 |
+
blue = "\033[1;34m"
|
| 87 |
+
yellow = "\033[1;33m"
|
| 88 |
+
green = "\033[1;32m"
|
| 89 |
+
red = "\033[1;31m"
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _print_banner() -> None:
|
| 93 |
+
banner = f"""
|
| 94 |
+
{blue}888 {gray} .d8888b.
|
| 95 |
+
{blue}888 {gray}d88P Y88b
|
| 96 |
+
{blue}888 {gray}888 888
|
| 97 |
+
{blue}888 .d88b. {gray}888 .d88b. 88888b.
|
| 98 |
+
{blue}888 d8P Y8b {gray}888 88888 d8P Y8b 888 "88b
|
| 99 |
+
{blue}888 88888888 {gray}888 888 88888888 888 888
|
| 100 |
+
{blue}888 Y8b. {gray}Y88b d88P Y8b. 888 888
|
| 101 |
+
{blue}88888888 "Y8888 {gray} "Y8888P88 "Y8888 888 888
|
| 102 |
+
|
| 103 |
+
legen {version} - github.com/matheusbach/legen{default}
|
| 104 |
+
python {sys.version}
|
| 105 |
+
"""
|
| 106 |
+
print(banner)
|
| 107 |
+
time.sleep(1)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 111 |
+
parser = argparse.ArgumentParser(prog="LeGen", description="Uses AI to locally transcribes speech from media files, generating subtitle files, translates the generated subtitles, inserts them into the mp4 container, and burns them directly into video",
|
| 112 |
+
argument_default=True, allow_abbrev=True, add_help=True, usage='LeGen -i INPUT_PATH [other options]')
|
| 113 |
+
parser.add_argument("-i", "--input_path",
|
| 114 |
+
help="Local media path or URL to download before processing", required=True, type=str)
|
| 115 |
+
parser.add_argument("--norm", default=False, action="store_true",
|
| 116 |
+
help="Normalize folder times and run vidqa on input_path before starting processing files")
|
| 117 |
+
parser.add_argument("-ts:e", "--transcription_engine", type=str, default="whisperx",
|
| 118 |
+
help="Transcription engine. Possible values: whisperx (default), whisper")
|
| 119 |
+
parser.add_argument("-ts:m", "--transcription_model", type=str, default="large-v3-turbo",
|
| 120 |
+
help="Path or name of the Whisper transcription model. A larger model will consume more resources and be slower, but with better transcription quality. Possible values: tiny, base, small, medium, large, turbo, large-v3-turbo (default) ...")
|
| 121 |
+
parser.add_argument("-ts:d", "--transcription_device", type=str, default="auto",
|
| 122 |
+
help="Device to run the transcription through Whisper. Possible values: auto (default), cpu, cuda")
|
| 123 |
+
parser.add_argument("-ts:c", "--transcription_compute_type", type=str, default="auto",
|
| 124 |
+
help="Quantization for the neural network. Possible values: auto (default), int8, int8_float32, int8_float16, int8_bfloat16, int16, float16, bfloat16, float32")
|
| 125 |
+
parser.add_argument("-ts:v", "--transcription_vad", type=str.lower, default="silero", choices=["pyannote", "silero"],
|
| 126 |
+
help="Voice activity detector to segment audio before transcription when using whisperx. Defaults to silero (CPU friendly).")
|
| 127 |
+
parser.add_argument("-ts:b", "--transcription_batch", type=int, default=4,
|
| 128 |
+
help="Number of simultaneous segments being transcribed. Higher values will speed up processing. If you have low RAM/VRAM, long duration media files or have buggy subtitles, reduce this value to avoid issues. Only works using transcription_engine whisperx. (default: 4)")
|
| 129 |
+
parser.add_argument("--translate", type=str, default="none",
|
| 130 |
+
help="Translate subtitles to language code if not the same as origin. (default: don't translate)")
|
| 131 |
+
parser.add_argument("--translate_engine", type=str, default="google",
|
| 132 |
+
help="Translation engine to use: google (default) or gemini")
|
| 133 |
+
parser.add_argument("--gemini_api_key", action="append", default=[], type=str,
|
| 134 |
+
help="Gemini API key. Repeat or separate by comma/line break to add multiple keys (required if --translate_engine=gemini)")
|
| 135 |
+
parser.add_argument("--tltw", action="store_true", default=False,
|
| 136 |
+
help="Generate a Gemini 'Too Long To Watch' summary from subtitles (requires --gemini_api_key)")
|
| 137 |
+
parser.add_argument("--output_tltw", type=Path, default=None,
|
| 138 |
+
help="Directory to save TLTW summaries. Defaults to the softsubs output folder")
|
| 139 |
+
parser.add_argument("--input_lang", type=str, default="auto",
|
| 140 |
+
help="Indicates (forces) the language of the voice in the input media (default: auto)")
|
| 141 |
+
parser.add_argument("-c:v", "--codec_video", type=str, default="h264", metavar="VIDEO_CODEC",
|
| 142 |
+
help="Target video codec. Can be used to set acceleration via GPU or another video API [codec_api], if supported (ffmpeg -encoders). Ex: h264, libx264, h264_vaapi, h264_nvenc, hevc, libx265 hevc_vaapi, hevc_nvenc, hevc_cuvid, hevc_qsv, hevc_amf (default: h264)")
|
| 143 |
+
parser.add_argument("-c:a", "--codec_audio", type=str, default="aac", metavar="AUDIO_CODEC",
|
| 144 |
+
help="Target audio codec. (default: aac). Ex: aac, libopus, mp3, vorbis")
|
| 145 |
+
parser.add_argument("-o:s", "--output_softsubs", default=None, type=Path,
|
| 146 |
+
help="Path to the folder or output file for the video files with embedded softsub (embedded in the mp4 container and .srt files). (default: softsubs_ + input_path)")
|
| 147 |
+
parser.add_argument("-o:h", "--output_hardsubs", default=None, type=Path,
|
| 148 |
+
help="Output folder path for video files with burned-in captions and embedded in the mp4 container. (default: hardsubs_ + input_path)")
|
| 149 |
+
parser.add_argument("-o:d", "--output_downloads", default=None, type=Path,
|
| 150 |
+
help="Destination folder for videos downloaded from URL inputs. (default: ./downloads)")
|
| 151 |
+
parser.add_argument("--overwrite", default=False, action="store_true",
|
| 152 |
+
help="Overwrite existing files in output directories")
|
| 153 |
+
parser.add_argument("-dl:rs", "--download_remote_subs", default=False, action="store_true",
|
| 154 |
+
help="When using a URL input, also download and embed remote subtitle tracks (disabled by default)")
|
| 155 |
+
parser.add_argument("--disable_srt", default=False, action="store_true",
|
| 156 |
+
help="Disable .srt file generation and don't insert subtitles in mp4 container of output_softsubs")
|
| 157 |
+
parser.add_argument("--subtitle_formats", type=str, default="srt",
|
| 158 |
+
help="Subtitle formats to export (separate multiple options with comma or space). Supported: srt, txt")
|
| 159 |
+
parser.add_argument("--disable_softsubs", default=False, action="store_true",
|
| 160 |
+
help="Don't insert subtitles in mp4 container of output_softsubs. This option continues generating .srt files")
|
| 161 |
+
parser.add_argument("--disable_hardsubs", default=False, action="store_true",
|
| 162 |
+
help="Disable subtitle burn in output_hardsubs")
|
| 163 |
+
parser.add_argument("--copy_files", default=False, action="store_true",
|
| 164 |
+
help="Copy other (non-video) files present in input directory to output directories. Only generate the subtitles and videos")
|
| 165 |
+
parser.add_argument(
|
| 166 |
+
"--process_input_subs",
|
| 167 |
+
"--process_srt_inputs",
|
| 168 |
+
dest="process_input_subs",
|
| 169 |
+
default=False,
|
| 170 |
+
action="store_true",
|
| 171 |
+
help="Also process existing .srt subtitle files found in the input path (translate/TLTW). When a subtitle matches a media filename, it is used instead of transcription.",
|
| 172 |
+
)
|
| 173 |
+
return parser
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def patch_torch_hub():
|
| 177 |
+
"""
|
| 178 |
+
Monkeypatch torch.hub.load to add retries for transient errors (like 503).
|
| 179 |
+
"""
|
| 180 |
+
try:
|
| 181 |
+
import torch.hub
|
| 182 |
+
import time
|
| 183 |
+
from urllib.error import HTTPError
|
| 184 |
+
|
| 185 |
+
original_load = torch.hub.load
|
| 186 |
+
|
| 187 |
+
def retrying_load(*args, **kwargs):
|
| 188 |
+
max_retries = 5
|
| 189 |
+
for attempt in range(max_retries):
|
| 190 |
+
try:
|
| 191 |
+
return original_load(*args, **kwargs)
|
| 192 |
+
except Exception as e:
|
| 193 |
+
# Check for 503 or other transient errors
|
| 194 |
+
is_transient = False
|
| 195 |
+
error_str = str(e)
|
| 196 |
+
if isinstance(e, HTTPError) and e.code in [500, 502, 503, 504]:
|
| 197 |
+
is_transient = True
|
| 198 |
+
elif "503" in error_str or "504" in error_str or "Connection reset" in error_str:
|
| 199 |
+
is_transient = True
|
| 200 |
+
|
| 201 |
+
if is_transient and attempt < max_retries - 1:
|
| 202 |
+
wait_time = 2 ** attempt
|
| 203 |
+
print(f"Download failed with {e}, retrying in {wait_time} seconds...")
|
| 204 |
+
time.sleep(wait_time)
|
| 205 |
+
continue
|
| 206 |
+
raise e
|
| 207 |
+
|
| 208 |
+
torch.hub.load = retrying_load
|
| 209 |
+
except ImportError:
|
| 210 |
+
pass
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def main(argv: Sequence[str] | None = None) -> int:
|
| 214 |
+
_print_banner()
|
| 215 |
+
parser = build_parser()
|
| 216 |
+
raw_argv = list(argv) if argv is not None else sys.argv[1:]
|
| 217 |
+
args = parser.parse_args(raw_argv if argv is not None else None)
|
| 218 |
+
|
| 219 |
+
input_path_raw = args.input_path
|
| 220 |
+
|
| 221 |
+
if looks_like_url(input_path_raw):
|
| 222 |
+
download_destination = args.output_downloads or Path.cwd() / "downloads"
|
| 223 |
+
args.output_downloads = Path(download_destination).expanduser().resolve()
|
| 224 |
+
try:
|
| 225 |
+
with time_task(message_start=f"\nDownloading media with yt-dlp to {gray}{args.output_downloads}{default}", end="\n"):
|
| 226 |
+
downloaded_files = download_utils.download_urls(
|
| 227 |
+
[input_path_raw],
|
| 228 |
+
args.output_downloads,
|
| 229 |
+
overwrite=args.overwrite,
|
| 230 |
+
download_remote_subs=args.download_remote_subs,
|
| 231 |
+
)
|
| 232 |
+
except FileNotFoundError as exc:
|
| 233 |
+
parser.error(str(exc))
|
| 234 |
+
except RuntimeError as exc:
|
| 235 |
+
parser.error(str(exc))
|
| 236 |
+
|
| 237 |
+
print(f"Downloaded {len(downloaded_files)} file(s) with {gray}yt-dlp{default}. Continuing with local processing from {gray}{args.output_downloads}{default}.")
|
| 238 |
+
args.input_path = args.output_downloads
|
| 239 |
+
else:
|
| 240 |
+
candidate_path = Path(input_path_raw).expanduser().resolve()
|
| 241 |
+
if not candidate_path.exists():
|
| 242 |
+
parser.error(f"Input path '{input_path_raw}' is neither an existing file/folder nor a downloadable URL.")
|
| 243 |
+
args.input_path = candidate_path
|
| 244 |
+
|
| 245 |
+
if args.output_downloads is not None:
|
| 246 |
+
args.output_downloads = Path(args.output_downloads).expanduser().resolve()
|
| 247 |
+
|
| 248 |
+
requested_formats = normalize_subtitle_formats(args.subtitle_formats)
|
| 249 |
+
unsupported_formats = [fmt for fmt in requested_formats if fmt not in SUPPORTED_SUBTITLE_FORMATS]
|
| 250 |
+
if unsupported_formats:
|
| 251 |
+
parser.error(f"Unsupported subtitle format(s): {', '.join(unsupported_formats)}. Supported formats: {', '.join(sorted(SUPPORTED_SUBTITLE_FORMATS))}")
|
| 252 |
+
|
| 253 |
+
if args.disable_srt:
|
| 254 |
+
requested_formats = [fmt for fmt in requested_formats if fmt != "srt"]
|
| 255 |
+
|
| 256 |
+
if not requested_formats and not args.disable_srt:
|
| 257 |
+
requested_formats = ["srt"]
|
| 258 |
+
|
| 259 |
+
args.subtitle_formats = requested_formats
|
| 260 |
+
args.disable_srt = "srt" not in args.subtitle_formats
|
| 261 |
+
args.export_txt = "txt" in args.subtitle_formats
|
| 262 |
+
|
| 263 |
+
def export_txt_if_requested(source_path: Path, target_path: Path):
|
| 264 |
+
if not args.export_txt:
|
| 265 |
+
return
|
| 266 |
+
if source_path is None:
|
| 267 |
+
return
|
| 268 |
+
if not args.overwrite and file_utils.file_is_valid(target_path):
|
| 269 |
+
return
|
| 270 |
+
subtitle_utils.export_plain_text_from_srt(source_path, target_path)
|
| 271 |
+
|
| 272 |
+
args.gemini_api_keys = normalize_api_keys(args.gemini_api_key)
|
| 273 |
+
args.gemini_api_key = args.gemini_api_keys[0] if args.gemini_api_keys else None
|
| 274 |
+
|
| 275 |
+
translate_engine_explicit = any(str(item).startswith("--translate_engine") for item in raw_argv)
|
| 276 |
+
if (
|
| 277 |
+
not translate_engine_explicit
|
| 278 |
+
and args.translate
|
| 279 |
+
and str(args.translate).lower() != "none"
|
| 280 |
+
and args.translate_engine == "google"
|
| 281 |
+
and args.gemini_api_keys
|
| 282 |
+
):
|
| 283 |
+
# If the user provided Gemini keys but did not explicitly choose an engine,
|
| 284 |
+
# prefer Gemini translation.
|
| 285 |
+
args.translate_engine = "gemini"
|
| 286 |
+
|
| 287 |
+
if args.tltw and not args.gemini_api_keys:
|
| 288 |
+
parser.error("Gemini API key is required for TLTW summaries. Provide --gemini_api_key.")
|
| 289 |
+
|
| 290 |
+
if args.translate_engine == "gemini" and not args.gemini_api_keys:
|
| 291 |
+
parser.error("Gemini API key is required when --translate_engine=gemini. Provide --gemini_api_key.")
|
| 292 |
+
|
| 293 |
+
if args.translate_engine == "gemini" and args.translate.lower() == 'pt':
|
| 294 |
+
args.translate = 'pt-BR'
|
| 295 |
+
if args.translate_engine == "google" and args.translate.lower() in ['pt', 'pt-br', 'pt-pt']:
|
| 296 |
+
args.translate = 'pt'
|
| 297 |
+
|
| 298 |
+
if not args.output_softsubs:
|
| 299 |
+
if args.input_path.is_file():
|
| 300 |
+
args.output_softsubs = Path(args.input_path.parent, "softsubs")
|
| 301 |
+
else:
|
| 302 |
+
args.output_softsubs = compatibility_path if (compatibility_path := Path(args.input_path.parent, "legen_srt_" + args.input_path.name)).exists() else Path(args.input_path.parent, "softsubs_" + args.input_path.name)
|
| 303 |
+
if not args.output_hardsubs:
|
| 304 |
+
if args.input_path.is_file():
|
| 305 |
+
args.output_hardsubs = Path(args.input_path.parent, "hardsubs")
|
| 306 |
+
else:
|
| 307 |
+
args.output_hardsubs = compatibility_path if (compatibility_path := Path(args.input_path.parent, "legen_burned_" + args.input_path.name)).exists() else Path(args.input_path.parent, "hardsubs_" + args.input_path.name)
|
| 308 |
+
|
| 309 |
+
if args.output_tltw:
|
| 310 |
+
args.output_tltw = Path(args.output_tltw).expanduser().resolve()
|
| 311 |
+
else:
|
| 312 |
+
args.output_tltw = args.output_softsubs
|
| 313 |
+
|
| 314 |
+
device_info = None
|
| 315 |
+
resolved_compute_type = None
|
| 316 |
+
|
| 317 |
+
if args.transcription_device == "auto":
|
| 318 |
+
# centralized, more robust device detection
|
| 319 |
+
from device_utils import select_torch_device
|
| 320 |
+
|
| 321 |
+
try:
|
| 322 |
+
device_info = select_torch_device(
|
| 323 |
+
preferred="auto",
|
| 324 |
+
model_name=args.transcription_model,
|
| 325 |
+
compute_type=args.transcription_compute_type,
|
| 326 |
+
)
|
| 327 |
+
except Exception as exc:
|
| 328 |
+
print(f"{yellow}Device detection failed ({exc}). Falling back to CPU.{default}")
|
| 329 |
+
torch_device = "cpu"
|
| 330 |
+
else:
|
| 331 |
+
torch_device = device_info.backend
|
| 332 |
+
resolved_compute_type = device_info.resolved_compute_type
|
| 333 |
+
for message in device_info.messages:
|
| 334 |
+
print(message)
|
| 335 |
+
for issue in device_info.issues:
|
| 336 |
+
print(f"{yellow}{issue}{default}")
|
| 337 |
+
for note in device_info.notes:
|
| 338 |
+
print(f"{gray}{note}{default}")
|
| 339 |
+
|
| 340 |
+
# If we have PyTorch and CUDA was selected, try to enable TF32 where
|
| 341 |
+
# possible for a small speedup on Ampere+ hardware. Keep this best-effort
|
| 342 |
+
# and ignore any failures.
|
| 343 |
+
try:
|
| 344 |
+
import torch
|
| 345 |
+
|
| 346 |
+
if torch_device.startswith("cuda"):
|
| 347 |
+
matmul_backend = getattr(torch.backends.cuda, "matmul", None)
|
| 348 |
+
if matmul_backend is not None and hasattr(matmul_backend, "fp32_precision"):
|
| 349 |
+
matmul_backend.fp32_precision = "tf32"
|
| 350 |
+
elif matmul_backend is not None and hasattr(matmul_backend, "allow_tf32"):
|
| 351 |
+
matmul_backend.allow_tf32 = True
|
| 352 |
+
|
| 353 |
+
cudnn_conv_backend = getattr(torch.backends.cudnn, "conv", None)
|
| 354 |
+
if cudnn_conv_backend is not None and hasattr(cudnn_conv_backend, "fp32_precision"):
|
| 355 |
+
cudnn_conv_backend.fp32_precision = "tf32"
|
| 356 |
+
elif hasattr(torch.backends.cudnn, "allow_tf32"):
|
| 357 |
+
torch.backends.cudnn.allow_tf32 = True
|
| 358 |
+
except Exception:
|
| 359 |
+
# ignore failures in optional PyTorch tuning
|
| 360 |
+
pass
|
| 361 |
+
else:
|
| 362 |
+
torch_device = str(args.transcription_device).lower()
|
| 363 |
+
|
| 364 |
+
if resolved_compute_type:
|
| 365 |
+
transcription_compute_type = resolved_compute_type
|
| 366 |
+
elif args.transcription_compute_type in {"auto", "default"}:
|
| 367 |
+
if torch_device.startswith("cuda"):
|
| 368 |
+
transcription_compute_type = "float16"
|
| 369 |
+
elif torch_device == "mps":
|
| 370 |
+
transcription_compute_type = "float16"
|
| 371 |
+
else:
|
| 372 |
+
transcription_compute_type = "float32"
|
| 373 |
+
else:
|
| 374 |
+
transcription_compute_type = args.transcription_compute_type
|
| 375 |
+
|
| 376 |
+
args.transcription_model = "large-v3" if args.transcription_model == "large" else args.transcription_model
|
| 377 |
+
|
| 378 |
+
if args.norm:
|
| 379 |
+
vidqa_executable = "vidqa"
|
| 380 |
+
local_vidqa = Path(sys.executable).parent / "vidqa"
|
| 381 |
+
if local_vidqa.exists() and os.access(local_vidqa, os.X_OK):
|
| 382 |
+
vidqa_executable = str(local_vidqa)
|
| 383 |
+
|
| 384 |
+
with time_task(message_start=f"Running {wblue}vidqa{default} and updating folder modifiation times in {gray}{args.input_path}{default}", end="\n"):
|
| 385 |
+
subprocess.run([vidqa_executable, "-i", args.input_path, "-m", "unique", "-fd",
|
| 386 |
+
Path(Path(getframeinfo(currentframe()).filename).resolve().parent, "vidqa_data")])
|
| 387 |
+
file_utils.update_folder_times(args.input_path)
|
| 388 |
+
|
| 389 |
+
with time_task(message_start=f"\nLoading {args.transcription_engine} model: {wblue}{args.transcription_model}{default} ({transcription_compute_type}) on {wblue}{torch_device}{default}", end="\n"):
|
| 390 |
+
if args.transcription_engine == 'whisperx':
|
| 391 |
+
patch_torch_hub()
|
| 392 |
+
import whisperx_legen_fork as whisperx
|
| 393 |
+
import whisperx_utils
|
| 394 |
+
|
| 395 |
+
whisper_model = whisperx.load_model(
|
| 396 |
+
whisper_arch=args.transcription_model,
|
| 397 |
+
device=torch_device,
|
| 398 |
+
compute_type=transcription_compute_type,
|
| 399 |
+
vad_method=args.transcription_vad,
|
| 400 |
+
asr_options={"repetition_penalty": 1, "prompt_reset_on_temperature": 0.5, "no_repeat_ngram_size": 2,},
|
| 401 |
+
)
|
| 402 |
+
elif args.transcription_engine == 'whisper':
|
| 403 |
+
import whisper
|
| 404 |
+
|
| 405 |
+
import whisper_utils
|
| 406 |
+
whisper_model = whisper.load_model(
|
| 407 |
+
name=args.transcription_model, device=torch_device, in_memory=True)
|
| 408 |
+
else:
|
| 409 |
+
raise ValueError(f'Unsupported transcription engine {args.transcription_engine}. Supported values: whisperx, whisper')
|
| 410 |
+
|
| 411 |
+
with time_task(message="⌛ Processing files for"):
|
| 412 |
+
# Pre-scan subtitle inputs so we can (a) use them instead of transcription
|
| 413 |
+
# for matching media, and (b) process standalone subtitle files.
|
| 414 |
+
srt_index: dict[tuple[Path, str], list[Path]] = {}
|
| 415 |
+
srt_linked_to_media: set[Path] = set()
|
| 416 |
+
|
| 417 |
+
if args.process_input_subs and args.input_path.is_dir():
|
| 418 |
+
media_keys: set[tuple[Path, str]] = set()
|
| 419 |
+
for media_path in args.input_path.rglob("*"):
|
| 420 |
+
if not media_path.is_file():
|
| 421 |
+
continue
|
| 422 |
+
ext = media_path.suffix.lower()
|
| 423 |
+
if ext not in (video_extensions | audio_extensions):
|
| 424 |
+
continue
|
| 425 |
+
rel_media = media_path.relative_to(args.input_path)
|
| 426 |
+
media_keys.add((rel_media.parent, rel_media.stem))
|
| 427 |
+
|
| 428 |
+
for srt_path in args.input_path.rglob("*.srt"):
|
| 429 |
+
if not srt_path.is_file():
|
| 430 |
+
continue
|
| 431 |
+
rel_srt = srt_path.relative_to(args.input_path)
|
| 432 |
+
base_stem, _ = split_lang_suffix(rel_srt.stem)
|
| 433 |
+
key = (rel_srt.parent, base_stem)
|
| 434 |
+
srt_index.setdefault(key, []).append(srt_path)
|
| 435 |
+
if key in media_keys:
|
| 436 |
+
srt_linked_to_media.add(srt_path)
|
| 437 |
+
|
| 438 |
+
# deterministic selection order
|
| 439 |
+
for key in list(srt_index.keys()):
|
| 440 |
+
srt_index[key] = sorted(srt_index[key])
|
| 441 |
+
|
| 442 |
+
path: Path
|
| 443 |
+
if args.input_path.is_file():
|
| 444 |
+
files_iterator = [args.input_path]
|
| 445 |
+
else:
|
| 446 |
+
files_iterator = (item for item in sorted(sorted(Path(args.input_path).rglob('*'), key=lambda x: x.stat().st_mtime), key=lambda x: len(x.parts)) if item.is_file())
|
| 447 |
+
|
| 448 |
+
for path in files_iterator:
|
| 449 |
+
if args.input_path.is_file():
|
| 450 |
+
rel_path = Path(path.name)
|
| 451 |
+
else:
|
| 452 |
+
rel_path = path.relative_to(args.input_path)
|
| 453 |
+
with time_task(message_start=f"\nProcessing {yellow}{rel_path.as_posix()}{default}", end="\n", message="⌚ Done in"):
|
| 454 |
+
try:
|
| 455 |
+
if path.suffix.lower() in video_extensions:
|
| 456 |
+
file_type = "video"
|
| 457 |
+
elif path.suffix.lower() in audio_extensions:
|
| 458 |
+
file_type = "audio"
|
| 459 |
+
elif args.process_input_subs and path.suffix.lower() == ".srt":
|
| 460 |
+
file_type = "subtitle"
|
| 461 |
+
else:
|
| 462 |
+
file_type = "other"
|
| 463 |
+
|
| 464 |
+
if file_type == "video" or file_type == "audio":
|
| 465 |
+
origin_media_path = path
|
| 466 |
+
dupe_filename = len(check_other_extensions(path, list(video_extensions | audio_extensions))) > 1
|
| 467 |
+
posfix_extension = path.suffix.lower().replace('.', '_') if dupe_filename else ''
|
| 468 |
+
|
| 469 |
+
softsub_video_dir = Path(args.output_softsubs, rel_path.parent)
|
| 470 |
+
burned_video_dir = Path(args.output_hardsubs, rel_path.parent)
|
| 471 |
+
softsub_video_path = Path(args.output_softsubs, rel_path.stem + posfix_extension + ".mp4")
|
| 472 |
+
hardsub_video_path = Path(burned_video_dir, rel_path.stem + posfix_extension + ".mp4")
|
| 473 |
+
subtitle_translated_path = Path(
|
| 474 |
+
softsub_video_dir, rel_path.stem + posfix_extension + f"_{args.translate.lower()}.srt")
|
| 475 |
+
subtitles_path = []
|
| 476 |
+
|
| 477 |
+
# If enabled, prefer an existing subtitle file matching this media name
|
| 478 |
+
linked_srt: Path | None = None
|
| 479 |
+
linked_srt_lang: str | None = None
|
| 480 |
+
if args.process_input_subs and args.input_path.is_dir():
|
| 481 |
+
candidates = srt_index.get((rel_path.parent, rel_path.stem), [])
|
| 482 |
+
if candidates:
|
| 483 |
+
linked_srt = candidates[0]
|
| 484 |
+
_, linked_srt_lang = split_lang_suffix(linked_srt.stem)
|
| 485 |
+
|
| 486 |
+
if linked_srt is not None:
|
| 487 |
+
if args.input_lang != "auto":
|
| 488 |
+
audio_language = args.input_lang
|
| 489 |
+
else:
|
| 490 |
+
audio_language = linked_srt_lang or "auto"
|
| 491 |
+
subtitle_transcribed_path = Path(
|
| 492 |
+
softsub_video_dir, rel_path.stem + posfix_extension + f"_{audio_language.lower()}.srt")
|
| 493 |
+
transcribed_srt_temp = file_utils.TempFile(
|
| 494 |
+
subtitle_transcribed_path, file_ext=".srt")
|
| 495 |
+
|
| 496 |
+
if file_utils.file_is_valid(subtitle_transcribed_path) and not args.overwrite:
|
| 497 |
+
print(f"Existing subtitle file {gray}{subtitle_transcribed_path}{default}. Skipping copy.")
|
| 498 |
+
else:
|
| 499 |
+
file_utils.copy_file_if_different(linked_srt, transcribed_srt_temp.getpath(), silent=True)
|
| 500 |
+
if not args.disable_srt:
|
| 501 |
+
transcribed_srt_temp.save()
|
| 502 |
+
else:
|
| 503 |
+
if args.input_lang == "auto":
|
| 504 |
+
audio_short_extracted = file_utils.TempFile(
|
| 505 |
+
None, file_ext=".wav")
|
| 506 |
+
ffmpeg_utils.extract_short_wav(
|
| 507 |
+
origin_media_path, audio_short_extracted.getpath())
|
| 508 |
+
print("Detecting audio language: ", end='', flush=True)
|
| 509 |
+
if args.transcription_engine == 'whisperx':
|
| 510 |
+
audio_language = whisperx_utils.detect_language(
|
| 511 |
+
whisper_model, audio_short_extracted.getpath())
|
| 512 |
+
if args.transcription_engine == 'whisper':
|
| 513 |
+
audio_language = whisper_utils.detect_language(
|
| 514 |
+
whisper_model, audio_short_extracted.getpath())
|
| 515 |
+
print(f"{gray}{audio_language}{default}")
|
| 516 |
+
|
| 517 |
+
audio_short_extracted.destroy()
|
| 518 |
+
else:
|
| 519 |
+
audio_language = args.input_lang
|
| 520 |
+
print(f"Forced input audio language: {gray}{audio_language}{default}")
|
| 521 |
+
subtitle_transcribed_path = Path(
|
| 522 |
+
softsub_video_dir, rel_path.stem + posfix_extension + f"_{audio_language.lower()}.srt")
|
| 523 |
+
transcribed_srt_temp = file_utils.TempFile(
|
| 524 |
+
subtitle_transcribed_path, file_ext=".srt")
|
| 525 |
+
if (file_utils.file_is_valid(subtitle_transcribed_path)) or ((args.disable_hardsubs or file_utils.file_is_valid(hardsub_video_path)) and (args.disable_srt or file_utils.file_is_valid(subtitle_transcribed_path))) and not args.overwrite:
|
| 526 |
+
print("Transcription is unnecessary. Skipping.")
|
| 527 |
+
else:
|
| 528 |
+
audio_extracted = file_utils.TempFile(None, file_ext=".wav")
|
| 529 |
+
ffmpeg_utils.extract_audio_wav(
|
| 530 |
+
origin_media_path, audio_extracted.getpath())
|
| 531 |
+
if args.transcription_engine == 'whisperx':
|
| 532 |
+
print(f"{wblue}Transcribing{default} with {gray}WhisperX{default}")
|
| 533 |
+
whisperx_utils.transcribe_audio(
|
| 534 |
+
whisper_model, audio_extracted.getpath(), transcribed_srt_temp.getpath(), audio_language, device=torch_device, batch_size=args.transcription_batch)
|
| 535 |
+
if args.transcription_engine == 'whisper':
|
| 536 |
+
print(f"{wblue}Transcribing{default} with {gray}Whisper{default}")
|
| 537 |
+
whisper_utils.transcribe_audio(
|
| 538 |
+
model=whisper_model, audio_path=audio_extracted.getpath(), srt_path=transcribed_srt_temp.getpath(), lang=audio_language, disable_fp16=False if transcription_compute_type == "float16" or transcription_compute_type == "fp16" else True)
|
| 539 |
+
|
| 540 |
+
audio_extracted.destroy()
|
| 541 |
+
if not args.disable_srt:
|
| 542 |
+
transcribed_srt_temp.save()
|
| 543 |
+
transcribed_srt_source_path = transcribed_srt_temp.getvalidpath()
|
| 544 |
+
if transcribed_srt_source_path:
|
| 545 |
+
subtitles_path.append(transcribed_srt_source_path)
|
| 546 |
+
if args.translate == "none":
|
| 547 |
+
translated_srt_source_path = None
|
| 548 |
+
elif audio_language != "auto" and args.translate == audio_language:
|
| 549 |
+
print("Translation is unnecessary because input and output language are the same. Skipping.")
|
| 550 |
+
translated_srt_source_path = None
|
| 551 |
+
elif (args.disable_hardsubs or file_utils.file_is_valid(hardsub_video_path)) and (args.disable_srt or (file_utils.file_is_valid(subtitle_translated_path) and file_utils.file_is_valid(subtitle_transcribed_path) and file_utils.file_is_valid(subtitle_translated_path))) and not args.overwrite:
|
| 552 |
+
print("Translation is unnecessary. Skipping.")
|
| 553 |
+
subtitles_path.insert(0, subtitle_translated_path)
|
| 554 |
+
translated_srt_source_path = subtitle_translated_path if file_utils.file_is_valid(subtitle_translated_path) else None
|
| 555 |
+
elif file_utils.file_is_valid(subtitle_translated_path):
|
| 556 |
+
print("Translated file found. Skipping translation.")
|
| 557 |
+
subtitles_path.insert(0, subtitle_translated_path)
|
| 558 |
+
translated_srt_source_path = subtitle_translated_path
|
| 559 |
+
elif transcribed_srt_temp.getvalidpath():
|
| 560 |
+
translated_srt_temp = file_utils.TempFile(
|
| 561 |
+
subtitle_translated_path, file_ext=".srt")
|
| 562 |
+
|
| 563 |
+
print(f"{wblue}Translating{default} with {gray}{args.translate_engine.capitalize()}{default} to {gray}{args.translate}{default}")
|
| 564 |
+
translate_utils.translate_srt_file(
|
| 565 |
+
transcribed_srt_temp.getvalidpath(),
|
| 566 |
+
translated_srt_temp.getpath(),
|
| 567 |
+
args.translate,
|
| 568 |
+
translate_engine=args.translate_engine,
|
| 569 |
+
gemini_api_keys=args.gemini_api_keys,
|
| 570 |
+
overwrite=args.overwrite
|
| 571 |
+
)
|
| 572 |
+
if not args.disable_srt:
|
| 573 |
+
translated_srt_temp.save()
|
| 574 |
+
|
| 575 |
+
translated_srt_source_path = translated_srt_temp.getvalidpath()
|
| 576 |
+
subtitles_path.insert(0, translated_srt_source_path)
|
| 577 |
+
else:
|
| 578 |
+
translated_srt_source_path = None
|
| 579 |
+
if args.export_txt and transcribed_srt_source_path:
|
| 580 |
+
export_txt_if_requested(transcribed_srt_source_path, subtitle_transcribed_path.with_suffix(".txt"))
|
| 581 |
+
if args.export_txt and translated_srt_source_path:
|
| 582 |
+
export_txt_if_requested(translated_srt_source_path, subtitle_translated_path.with_suffix(".txt"))
|
| 583 |
+
if args.tltw:
|
| 584 |
+
summary_source_path = translated_srt_source_path or transcribed_srt_source_path
|
| 585 |
+
summary_language = "auto-detect" if audio_language == "auto" else audio_language
|
| 586 |
+
if summary_source_path and translated_srt_source_path and summary_source_path == translated_srt_source_path and args.translate.lower() != "none":
|
| 587 |
+
summary_language = args.translate
|
| 588 |
+
|
| 589 |
+
if summary_source_path:
|
| 590 |
+
summary_output_dir = Path(args.output_tltw, rel_path.parent)
|
| 591 |
+
summary_filename = f"{rel_path.stem + posfix_extension}_tltw_{str(summary_language).lower()}.md"
|
| 592 |
+
summary_output_path = summary_output_dir / summary_filename
|
| 593 |
+
|
| 594 |
+
if file_utils.file_is_valid(summary_output_path) and not args.overwrite:
|
| 595 |
+
print(f"Existing TLTW summary {gray}{summary_output_path}{default}. Skipping.")
|
| 596 |
+
else:
|
| 597 |
+
print(f"{wblue}Generating TLTW summary{default} with {gray}Gemini{default}")
|
| 598 |
+
generate_tltw(
|
| 599 |
+
GeminiSummaryConfig(
|
| 600 |
+
api_keys=args.gemini_api_keys,
|
| 601 |
+
subtitle_file=summary_source_path,
|
| 602 |
+
output_file=summary_output_path,
|
| 603 |
+
language=summary_language,
|
| 604 |
+
)
|
| 605 |
+
)
|
| 606 |
+
else:
|
| 607 |
+
print("No subtitles available for TLTW summary. Skipping.")
|
| 608 |
+
if not args.disable_softsubs:
|
| 609 |
+
if file_utils.file_is_valid(softsub_video_path) and not args.overwrite:
|
| 610 |
+
print(f"Existing video file {gray}{softsub_video_path}{default}. Skipping subtitle insert")
|
| 611 |
+
else:
|
| 612 |
+
video_softsubs_temp = file_utils.TempFile(
|
| 613 |
+
softsub_video_path, file_ext=".mp4")
|
| 614 |
+
|
| 615 |
+
print(f"{wblue}Inserting subtitle{default} in mp4 container using {gray}FFmpeg{default}")
|
| 616 |
+
ffmpeg_utils.insert_subtitle(input_media_path=origin_media_path, subtitles_path=subtitles_path,
|
| 617 |
+
burn_subtitles=False, output_video_path=video_softsubs_temp.getpath(),
|
| 618 |
+
codec_video=args.codec_video, codec_audio=args.codec_audio)
|
| 619 |
+
video_softsubs_temp.save()
|
| 620 |
+
if not args.disable_hardsubs:
|
| 621 |
+
if file_utils.file_is_valid(hardsub_video_path) and not args.overwrite:
|
| 622 |
+
print(f"Existing video file {gray}{hardsub_video_path}{default}. Skipping subtitle burn")
|
| 623 |
+
else:
|
| 624 |
+
video_hardsubs_temp = file_utils.TempFile(
|
| 625 |
+
hardsub_video_path, file_ext=".mp4")
|
| 626 |
+
print(f"{wblue}Inserting subtitle{default} in mp4 container and {wblue}burning{default} using {gray}FFmpeg{default}")
|
| 627 |
+
ffmpeg_utils.insert_subtitle(input_media_path=origin_media_path, subtitles_path=subtitles_path,
|
| 628 |
+
burn_subtitles=True, output_video_path=video_hardsubs_temp.getpath(),
|
| 629 |
+
codec_video=args.codec_video, codec_audio=args.codec_audio)
|
| 630 |
+
video_hardsubs_temp.save()
|
| 631 |
+
elif file_type == "subtitle":
|
| 632 |
+
# Standalone subtitle files (not linked to any media) can be translated and summarized.
|
| 633 |
+
if args.input_path.is_dir() and path in srt_linked_to_media:
|
| 634 |
+
print("Subtitle matches a media file; it will be used during media processing. Skipping standalone processing.")
|
| 635 |
+
continue
|
| 636 |
+
|
| 637 |
+
# Derive base stem and language from filename suffix convention: name_lang.srt
|
| 638 |
+
base_stem, lang_from_suffix = split_lang_suffix(rel_path.stem)
|
| 639 |
+
if args.input_lang != "auto":
|
| 640 |
+
subtitle_language = args.input_lang
|
| 641 |
+
else:
|
| 642 |
+
subtitle_language = lang_from_suffix or "auto"
|
| 643 |
+
|
| 644 |
+
softsub_dir = Path(args.output_softsubs, rel_path.parent)
|
| 645 |
+
subtitle_transcribed_path = Path(softsub_dir, f"{base_stem}_{subtitle_language.lower()}.srt")
|
| 646 |
+
subtitle_translated_path = Path(softsub_dir, f"{base_stem}_{args.translate.lower()}.srt")
|
| 647 |
+
|
| 648 |
+
transcribed_srt_temp = file_utils.TempFile(subtitle_transcribed_path, file_ext=".srt")
|
| 649 |
+
if file_utils.file_is_valid(subtitle_transcribed_path) and not args.overwrite:
|
| 650 |
+
print(f"Existing subtitle file {gray}{subtitle_transcribed_path}{default}. Skipping copy.")
|
| 651 |
+
transcribed_srt_source_path = subtitle_transcribed_path
|
| 652 |
+
else:
|
| 653 |
+
file_utils.copy_file_if_different(path, transcribed_srt_temp.getpath(), silent=True)
|
| 654 |
+
if not args.disable_srt:
|
| 655 |
+
transcribed_srt_temp.save()
|
| 656 |
+
transcribed_srt_source_path = transcribed_srt_temp.getvalidpath()
|
| 657 |
+
|
| 658 |
+
if args.translate == "none":
|
| 659 |
+
translated_srt_source_path = None
|
| 660 |
+
elif subtitle_language != "auto" and args.translate == subtitle_language:
|
| 661 |
+
print("Translation is unnecessary because input and output language are the same. Skipping.")
|
| 662 |
+
translated_srt_source_path = None
|
| 663 |
+
elif file_utils.file_is_valid(subtitle_translated_path) and not args.overwrite:
|
| 664 |
+
print("Translated file found. Skipping translation.")
|
| 665 |
+
translated_srt_source_path = subtitle_translated_path
|
| 666 |
+
elif transcribed_srt_source_path:
|
| 667 |
+
translated_srt_temp = file_utils.TempFile(subtitle_translated_path, file_ext=".srt")
|
| 668 |
+
print(f"{wblue}Translating{default} with {gray}{args.translate_engine.capitalize()}{default} to {gray}{args.translate}{default}")
|
| 669 |
+
translate_utils.translate_srt_file(
|
| 670 |
+
transcribed_srt_source_path,
|
| 671 |
+
translated_srt_temp.getpath(),
|
| 672 |
+
args.translate,
|
| 673 |
+
translate_engine=args.translate_engine,
|
| 674 |
+
gemini_api_keys=args.gemini_api_keys,
|
| 675 |
+
overwrite=args.overwrite,
|
| 676 |
+
)
|
| 677 |
+
if not args.disable_srt:
|
| 678 |
+
translated_srt_temp.save()
|
| 679 |
+
translated_srt_source_path = translated_srt_temp.getvalidpath()
|
| 680 |
+
else:
|
| 681 |
+
translated_srt_source_path = None
|
| 682 |
+
|
| 683 |
+
if args.export_txt and transcribed_srt_source_path:
|
| 684 |
+
export_txt_if_requested(transcribed_srt_source_path, subtitle_transcribed_path.with_suffix(".txt"))
|
| 685 |
+
if args.export_txt and translated_srt_source_path:
|
| 686 |
+
export_txt_if_requested(translated_srt_source_path, subtitle_translated_path.with_suffix(".txt"))
|
| 687 |
+
|
| 688 |
+
if args.tltw:
|
| 689 |
+
summary_source_path = translated_srt_source_path or transcribed_srt_source_path
|
| 690 |
+
summary_language = "auto-detect" if subtitle_language == "auto" else subtitle_language
|
| 691 |
+
if summary_source_path and translated_srt_source_path and summary_source_path == translated_srt_source_path and args.translate.lower() != "none":
|
| 692 |
+
summary_language = args.translate
|
| 693 |
+
|
| 694 |
+
if summary_source_path:
|
| 695 |
+
summary_output_dir = Path(args.output_tltw, rel_path.parent)
|
| 696 |
+
summary_filename = f"{base_stem}_tltw_{str(summary_language).lower()}.md"
|
| 697 |
+
summary_output_path = summary_output_dir / summary_filename
|
| 698 |
+
|
| 699 |
+
if file_utils.file_is_valid(summary_output_path) and not args.overwrite:
|
| 700 |
+
print(f"Existing TLTW summary {gray}{summary_output_path}{default}. Skipping.")
|
| 701 |
+
else:
|
| 702 |
+
print(f"{wblue}Generating TLTW summary{default} with {gray}Gemini{default}")
|
| 703 |
+
generate_tltw(
|
| 704 |
+
GeminiSummaryConfig(
|
| 705 |
+
api_keys=args.gemini_api_keys,
|
| 706 |
+
subtitle_file=summary_source_path,
|
| 707 |
+
output_file=summary_output_path,
|
| 708 |
+
language=summary_language,
|
| 709 |
+
)
|
| 710 |
+
)
|
| 711 |
+
else:
|
| 712 |
+
print("No subtitles available for TLTW summary. Skipping.")
|
| 713 |
+
else:
|
| 714 |
+
print("not a video file")
|
| 715 |
+
if args.copy_files:
|
| 716 |
+
if not args.disable_srt:
|
| 717 |
+
file_utils.copy_file_if_different(path, Path(
|
| 718 |
+
args.output_softsubs, rel_path))
|
| 719 |
+
if not args.disable_hardsubs:
|
| 720 |
+
file_utils.copy_file_if_different(path, Path(
|
| 721 |
+
args.output_hardsubs, rel_path))
|
| 722 |
+
except Exception as e: # noqa: BLE001
|
| 723 |
+
file = path.as_posix()
|
| 724 |
+
print(f"{red}ERROR !!!{default} {file}")
|
| 725 |
+
print(f"{yellow}check legen-errors.txt for details{default}")
|
| 726 |
+
current_time = time.strftime("%y/%m/%d %H:%M:%S", time.localtime())
|
| 727 |
+
|
| 728 |
+
error_message = f"[{current_time}] {file}: {type(e).__name__}: {str(e)}"
|
| 729 |
+
with open(Path(Path(getframeinfo(currentframe()).filename).resolve().parent, "legen-errors.txt"), "a") as f:
|
| 730 |
+
f.write(error_message + "\n")
|
| 731 |
+
f.close()
|
| 732 |
+
|
| 733 |
+
print("Deleting temp folder")
|
| 734 |
+
file_utils.delete_folder(
|
| 735 |
+
Path(Path(getframeinfo(currentframe()).filename).resolve().parent, "temp"))
|
| 736 |
+
|
| 737 |
+
print(f"{green}Tasks done!{default}")
|
| 738 |
+
return 0
|
| 739 |
+
|
| 740 |
+
|
| 741 |
+
if __name__ == "__main__": # pragma: no cover
|
| 742 |
+
raise SystemExit(main())
|
legen_cli.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Console entry point for the LeGen CLI."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import importlib
|
| 5 |
+
import sys
|
| 6 |
+
from typing import Sequence
|
| 7 |
+
|
| 8 |
+
_LEGACY_WARNING = (
|
| 9 |
+
"Warning: running legacy LeGen CLI fallback because the installed package "
|
| 10 |
+
"does not expose a 'main' entrypoint. Please upgrade to the latest release."
|
| 11 |
+
)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _restore_sys_argv(original: list[str] | None) -> None:
|
| 15 |
+
if original is not None:
|
| 16 |
+
sys.argv = original
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def main(argv: Sequence[str] | None = None) -> None:
|
| 20 |
+
"""Run the packaged LeGen CLI and propagate its exit code."""
|
| 21 |
+
|
| 22 |
+
forwarded_args = list(argv) if argv is not None else None
|
| 23 |
+
original_sys_argv: list[str] | None = None
|
| 24 |
+
|
| 25 |
+
if forwarded_args is not None:
|
| 26 |
+
# Mirror invocation semantics for callers that provide their own argv.
|
| 27 |
+
original_sys_argv = sys.argv.copy()
|
| 28 |
+
program_name = original_sys_argv[0] if original_sys_argv else "legen"
|
| 29 |
+
sys.argv = [program_name, *map(str, forwarded_args)]
|
| 30 |
+
|
| 31 |
+
try:
|
| 32 |
+
module = importlib.import_module("legen")
|
| 33 |
+
except Exception:
|
| 34 |
+
_restore_sys_argv(original_sys_argv)
|
| 35 |
+
raise
|
| 36 |
+
|
| 37 |
+
entrypoint = getattr(module, "main", None)
|
| 38 |
+
if callable(entrypoint):
|
| 39 |
+
try:
|
| 40 |
+
exit_code = entrypoint(forwarded_args)
|
| 41 |
+
finally:
|
| 42 |
+
_restore_sys_argv(original_sys_argv)
|
| 43 |
+
|
| 44 |
+
if exit_code not in (None, 0):
|
| 45 |
+
sys.exit(exit_code)
|
| 46 |
+
return
|
| 47 |
+
|
| 48 |
+
_restore_sys_argv(original_sys_argv)
|
| 49 |
+
|
| 50 |
+
# Legacy fallback: old packaged versions executed their CLI logic on import.
|
| 51 |
+
# At this point the module has already run using sys.argv, so avoid crashing
|
| 52 |
+
# with ImportError and exit gracefully instead.
|
| 53 |
+
print(_LEGACY_WARNING, file=sys.stderr)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
if __name__ == "__main__": # pragma: no cover
|
| 57 |
+
main()
|
pyproject.toml
CHANGED
|
@@ -1,40 +1,76 @@
|
|
| 1 |
-
[build-system]
|
| 2 |
-
requires = ["setuptools", "
|
| 3 |
-
build-backend = "setuptools.build_meta"
|
| 4 |
-
|
| 5 |
-
[project]
|
| 6 |
-
name = "
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
"
|
| 21 |
-
"
|
| 22 |
-
"
|
| 23 |
-
"
|
| 24 |
-
"
|
| 25 |
-
"
|
| 26 |
-
"
|
| 27 |
-
"
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
"
|
| 32 |
-
"
|
| 33 |
-
"
|
| 34 |
-
"
|
| 35 |
-
"
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=68", "wheel"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "legen"
|
| 7 |
+
version = "0.20.0"
|
| 8 |
+
description = "Powerfull toolkit that locally transcribes, translates, and masters subtitles for your media"
|
| 9 |
+
readme = { file = "README.md", content-type = "text/markdown" }
|
| 10 |
+
requires-python = ">=3.9,<3.13"
|
| 11 |
+
license = "GPL-3.0-only"
|
| 12 |
+
license-files = ["LICENSE"]
|
| 13 |
+
authors = [
|
| 14 |
+
{ name = "Matheus Bach" }
|
| 15 |
+
]
|
| 16 |
+
keywords = ["subtitles", "whisper", "translation", "ffmpeg", "yt-dlp"]
|
| 17 |
+
classifiers = [
|
| 18 |
+
"Development Status :: 4 - Beta",
|
| 19 |
+
"Environment :: Console",
|
| 20 |
+
"Intended Audience :: End Users/Desktop",
|
| 21 |
+
"Intended Audience :: Developers",
|
| 22 |
+
"Programming Language :: Python :: 3",
|
| 23 |
+
"Programming Language :: Python :: 3.9",
|
| 24 |
+
"Programming Language :: Python :: 3.10",
|
| 25 |
+
"Programming Language :: Python :: 3.11",
|
| 26 |
+
"Programming Language :: Python :: 3.12",
|
| 27 |
+
"Topic :: Multimedia :: Video"
|
| 28 |
+
]
|
| 29 |
+
dependencies = [
|
| 30 |
+
"deep-translator",
|
| 31 |
+
"ffmpeg-progress-yield",
|
| 32 |
+
"openai-whisper",
|
| 33 |
+
"pysrt",
|
| 34 |
+
"torch",
|
| 35 |
+
"torchaudio<2.9",
|
| 36 |
+
"tqdm",
|
| 37 |
+
"whisper",
|
| 38 |
+
"vidqa",
|
| 39 |
+
"gemini-srt-translator",
|
| 40 |
+
"google-generativeai",
|
| 41 |
+
"yt-dlp",
|
| 42 |
+
"gradio>=3.36.1",
|
| 43 |
+
"whisperx-legen-fork",
|
| 44 |
+
]
|
| 45 |
+
|
| 46 |
+
[project.optional-dependencies]
|
| 47 |
+
dev = [
|
| 48 |
+
"pytest>=8"
|
| 49 |
+
]
|
| 50 |
+
|
| 51 |
+
[project.urls]
|
| 52 |
+
Homepage = "https://github.com/matheusbach/legen"
|
| 53 |
+
"Issue Tracker" = "https://github.com/matheusbach/legen/issues"
|
| 54 |
+
|
| 55 |
+
[project.scripts]
|
| 56 |
+
legen = "legen_cli:main"
|
| 57 |
+
legen-translate = "translate_utils:main"
|
| 58 |
+
|
| 59 |
+
[tool.setuptools]
|
| 60 |
+
py-modules = [
|
| 61 |
+
"download_utils",
|
| 62 |
+
"ffmpeg_utils",
|
| 63 |
+
"file_utils",
|
| 64 |
+
"gemini_utils",
|
| 65 |
+
"device_utils",
|
| 66 |
+
"legen",
|
| 67 |
+
"legen_cli",
|
| 68 |
+
"subtitle_utils",
|
| 69 |
+
"translate_utils",
|
| 70 |
+
"utils",
|
| 71 |
+
"whisper_utils",
|
| 72 |
+
"whisperx_utils"
|
| 73 |
+
]
|
| 74 |
+
|
| 75 |
+
[tool.uv.sources]
|
| 76 |
+
whisperx-legen-fork = { git = "https://github.com/matheusbach/whisperX.git" }
|
pytest.ini
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[pytest]
|
| 2 |
+
testpaths = tests
|
| 3 |
+
python_files = test_*.py
|
| 4 |
+
norecursedirs = tmp temp downloads hardsubs_m softsubs_m m docs
|
| 5 |
+
addopts = -ra
|
requirements.txt
CHANGED
|
@@ -1,11 +1,13 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
pysrt
|
| 5 |
-
torch
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
|
|
|
|
|
|
|
|
| 1 |
+
deep-translator
|
| 2 |
+
ffmpeg-progress-yield
|
| 3 |
+
openai-whisper
|
| 4 |
+
pysrt
|
| 5 |
+
torch
|
| 6 |
+
torchaudio<2.9
|
| 7 |
+
tqdm
|
| 8 |
+
whisper
|
| 9 |
+
vidqa
|
| 10 |
+
whisperx-legen-fork
|
| 11 |
+
gemini-srt-translator
|
| 12 |
+
google-generativeai
|
| 13 |
+
yt-dlp
|
start.bat
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
@echo off
|
| 2 |
+
if not exist ".venv" (
|
| 3 |
+
echo Virtual environment not found. Please run install.bat first.
|
| 4 |
+
pause
|
| 5 |
+
exit /b
|
| 6 |
+
)
|
| 7 |
+
|
| 8 |
+
echo Activate venv...
|
| 9 |
+
call .venv\Scripts\activate
|
| 10 |
+
|
| 11 |
+
echo Starting LeGen WebUI...
|
| 12 |
+
python app.py
|
| 13 |
+
pause
|
subtitle_utils.py
CHANGED
|
@@ -1,171 +1,226 @@
|
|
| 1 |
-
import
|
| 2 |
-
import
|
| 3 |
-
import
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
import
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import atexit
|
| 2 |
+
import os
|
| 3 |
+
import re
|
| 4 |
+
import tkinter as tk
|
| 5 |
+
import tkinter.font as tk_font
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import pysrt
|
| 9 |
+
|
| 10 |
+
_STRING_WIDTH_ROOT = None
|
| 11 |
+
_FONT_CACHE = {}
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _destroy_string_width_root():
|
| 15 |
+
global _STRING_WIDTH_ROOT
|
| 16 |
+
if _STRING_WIDTH_ROOT is None:
|
| 17 |
+
return
|
| 18 |
+
try:
|
| 19 |
+
_STRING_WIDTH_ROOT.destroy()
|
| 20 |
+
except Exception:
|
| 21 |
+
pass
|
| 22 |
+
_STRING_WIDTH_ROOT = None
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _ensure_string_width_font(font_name: str, font_size: int) -> tk_font.Font:
|
| 26 |
+
global _STRING_WIDTH_ROOT
|
| 27 |
+
if _STRING_WIDTH_ROOT is None:
|
| 28 |
+
_STRING_WIDTH_ROOT = tk.Tk()
|
| 29 |
+
_STRING_WIDTH_ROOT.withdraw()
|
| 30 |
+
atexit.register(_destroy_string_width_root)
|
| 31 |
+
|
| 32 |
+
cache_key = (font_name, font_size)
|
| 33 |
+
font = _FONT_CACHE.get(cache_key)
|
| 34 |
+
if font is None:
|
| 35 |
+
font = tk_font.Font(root=_STRING_WIDTH_ROOT, family=font_name, size=font_size, weight="bold")
|
| 36 |
+
_FONT_CACHE[cache_key] = font
|
| 37 |
+
return font
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _collect_plain_text(subtitles: pysrt.SubRipFile) -> str:
|
| 41 |
+
texts = []
|
| 42 |
+
for item in subtitles:
|
| 43 |
+
normalized = " ".join(item.text.strip().split())
|
| 44 |
+
if normalized:
|
| 45 |
+
texts.append(normalized)
|
| 46 |
+
return " ".join(texts).strip()
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def export_plain_text_from_srt(source, output_path: Path) -> str:
|
| 50 |
+
"""Save subtitles as a single-line TXT without timestamps or line breaks."""
|
| 51 |
+
if isinstance(source, pysrt.SubRipFile):
|
| 52 |
+
subtitles = source
|
| 53 |
+
else:
|
| 54 |
+
subtitles = pysrt.open(source, encoding="utf-8")
|
| 55 |
+
|
| 56 |
+
plain_text = _collect_plain_text(subtitles)
|
| 57 |
+
|
| 58 |
+
os.makedirs(output_path.parent, exist_ok=True)
|
| 59 |
+
with open(output_path, "w", encoding="utf-8") as f:
|
| 60 |
+
f.write(plain_text)
|
| 61 |
+
|
| 62 |
+
return plain_text
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def SaveSegmentsToSrt(segments: list, output_path: Path):
|
| 66 |
+
# Create the subtitle file
|
| 67 |
+
subs = pysrt.SubRipFile()
|
| 68 |
+
sub_idx = 1
|
| 69 |
+
|
| 70 |
+
for i in range(len(segments)):
|
| 71 |
+
start_time = segments[i]["start"]
|
| 72 |
+
end_time = segments[i]["end"]
|
| 73 |
+
duration = end_time - start_time
|
| 74 |
+
timestamp = f"{start_time:.3f} - {end_time:.3f}"
|
| 75 |
+
text = segments[i]["text"]
|
| 76 |
+
|
| 77 |
+
sub = pysrt.SubRipItem(index=sub_idx, start=pysrt.SubRipTime(seconds=start_time),
|
| 78 |
+
end=pysrt.SubRipTime(seconds=end_time), text=text)
|
| 79 |
+
subs.append(sub)
|
| 80 |
+
sub_idx += 1
|
| 81 |
+
|
| 82 |
+
# make dir and save .srt
|
| 83 |
+
os.makedirs(output_path.parent, exist_ok=True)
|
| 84 |
+
subs.save(output_path)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def string_width(text, font_name="Jost", font_size=18):
|
| 88 |
+
"""
|
| 89 |
+
Determines the width of a string using tkinter.
|
| 90 |
+
"""
|
| 91 |
+
tries_remaining = 5
|
| 92 |
+
|
| 93 |
+
while (tries_remaining > 0):
|
| 94 |
+
tries_remaining -= 1
|
| 95 |
+
try:
|
| 96 |
+
font = _ensure_string_width_font(font_name, font_size)
|
| 97 |
+
width = font.measure(text)
|
| 98 |
+
return width
|
| 99 |
+
except Exception:
|
| 100 |
+
_destroy_string_width_root()
|
| 101 |
+
pass
|
| 102 |
+
|
| 103 |
+
# all failed, return 60% of height per char
|
| 104 |
+
return len(text) * font_size * 0.60
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def is_punctuation_end(word):
|
| 108 |
+
"""Verifica se a palavra termina com uma pontuação."""
|
| 109 |
+
return any(word.endswith(punct) for punct in ['.', ',', '!', '?', ':', ';'])
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def split_segments(segments, max_width_px=1440, font_name="Jost", font_size=18):
|
| 113 |
+
"""
|
| 114 |
+
Split segments based on the max width provided.
|
| 115 |
+
"""
|
| 116 |
+
new_segments = []
|
| 117 |
+
for segment in segments:
|
| 118 |
+
words = segment['words']
|
| 119 |
+
current_words = []
|
| 120 |
+
current_width = 0
|
| 121 |
+
|
| 122 |
+
for word in words:
|
| 123 |
+
# Calculate the width with a space after the word
|
| 124 |
+
added_width = string_width(
|
| 125 |
+
word['word'] + " ", font_name, font_size)
|
| 126 |
+
isolated_sentence_ending = is_punctuation_end(word['word']) and not (
|
| 127 |
+
current_words and is_punctuation_end(current_words[-1]['word']))
|
| 128 |
+
possible_logical_break_point = len(current_words) >= 2 and len(
|
| 129 |
+
current_words[-1]['word']) <= 3 and not len(current_words[-2]['word']) <= 3
|
| 130 |
+
|
| 131 |
+
if (current_width + added_width < max_width_px) or len(current_words) == 0 or isolated_sentence_ending or possible_logical_break_point:
|
| 132 |
+
current_words.append(word)
|
| 133 |
+
current_width += added_width
|
| 134 |
+
else:
|
| 135 |
+
new_segments.append({
|
| 136 |
+
'text': ' '.join(word['word'] for word in current_words),
|
| 137 |
+
'start': next((word['start'] for word in current_words if 'start' in word), segment['start']),
|
| 138 |
+
'end': next((word['end'] for word in reversed(current_words) if 'end' in word), segment['end']),
|
| 139 |
+
'words': current_words.copy()
|
| 140 |
+
})
|
| 141 |
+
current_words = [word]
|
| 142 |
+
current_width = added_width
|
| 143 |
+
|
| 144 |
+
# For any remaining words
|
| 145 |
+
if current_words:
|
| 146 |
+
new_segments.append({
|
| 147 |
+
'text': ' '.join(word['word'] for word in current_words),
|
| 148 |
+
'start': next((word['start'] for word in current_words if 'start' in word), segment['start']),
|
| 149 |
+
'end': next((word['end'] for word in reversed(current_words) if 'end' in word), segment['end']),
|
| 150 |
+
'words': current_words
|
| 151 |
+
})
|
| 152 |
+
|
| 153 |
+
return new_segments
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def split_string_to_max_lines(text, max_width=720, max_lines=2, font_name="Jost", font_size=18):
|
| 157 |
+
threshold = max_width * 0.8
|
| 158 |
+
total_text_width = string_width(text, font_name, font_size)
|
| 159 |
+
|
| 160 |
+
if total_text_width <= threshold or max_lines < 2:
|
| 161 |
+
return [text]
|
| 162 |
+
|
| 163 |
+
words = text.split()
|
| 164 |
+
lines = []
|
| 165 |
+
current_line_words = []
|
| 166 |
+
current_line_width = 0
|
| 167 |
+
|
| 168 |
+
for i, word in enumerate(words):
|
| 169 |
+
word_width = string_width(word + ' ', font_name, font_size)
|
| 170 |
+
isolated_sentence_ending = is_punctuation_end(word) and not (
|
| 171 |
+
current_line_words and is_punctuation_end(current_line_words[-1]))
|
| 172 |
+
possible_logical_break_point = len(current_line_words) >= 2 and len(
|
| 173 |
+
current_line_words[-1]) <= 3 and not len(current_line_words[-2]) <= 3
|
| 174 |
+
|
| 175 |
+
if current_line_width + word_width < total_text_width / max_lines or len(current_line_words) == 0 or isolated_sentence_ending or possible_logical_break_point:
|
| 176 |
+
current_line_words.append(word)
|
| 177 |
+
current_line_width += word_width
|
| 178 |
+
else:
|
| 179 |
+
lines.append(' '.join(current_line_words))
|
| 180 |
+
current_line_words = [word]
|
| 181 |
+
current_line_width = word_width
|
| 182 |
+
|
| 183 |
+
if len(lines) == max_lines - 1:
|
| 184 |
+
remaining_words = words[i:]
|
| 185 |
+
lines.append(' '.join(remaining_words))
|
| 186 |
+
break
|
| 187 |
+
|
| 188 |
+
if current_line_words and len(lines) < max_lines:
|
| 189 |
+
lines.append(' '.join(current_line_words))
|
| 190 |
+
|
| 191 |
+
return lines
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def adjust_times(segments, extra_end_time=1.0):
|
| 195 |
+
for i in range(len(segments) - 1): # We don't need to check the last segment
|
| 196 |
+
current_end = segments[i]['end']
|
| 197 |
+
next_start = segments[i + 1]['start']
|
| 198 |
+
|
| 199 |
+
gap = next_start - current_end
|
| 200 |
+
|
| 201 |
+
# If the gap is more than 1.5 + extra_end_time
|
| 202 |
+
if gap > 1.5 + extra_end_time:
|
| 203 |
+
segments[i]['end'] = current_end + extra_end_time
|
| 204 |
+
|
| 205 |
+
# If the gap is less than 1.5 + extra_end_time
|
| 206 |
+
elif gap < 1.5 + extra_end_time:
|
| 207 |
+
segments[i]['end'] = next_start
|
| 208 |
+
|
| 209 |
+
return segments
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def format_segments(segments: list, max_line_width_px: int = 380, max_lines_per_segment: int = 2):
|
| 213 |
+
print('Formatting segments...', end='', flush=True)
|
| 214 |
+
|
| 215 |
+
segments = split_segments(
|
| 216 |
+
segments, max_line_width_px * max_lines_per_segment)
|
| 217 |
+
|
| 218 |
+
for segment in segments:
|
| 219 |
+
segment["text"] = "\n".join(split_string_to_max_lines(
|
| 220 |
+
text=segment["text"], max_width=max_line_width_px, max_lines=max_lines_per_segment))
|
| 221 |
+
|
| 222 |
+
segments = adjust_times(segments)
|
| 223 |
+
|
| 224 |
+
print('\r ', end='\r', flush=True)
|
| 225 |
+
|
| 226 |
+
return segments
|
translate_utils.py
CHANGED
|
@@ -1,232 +1,438 @@
|
|
| 1 |
-
import
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
import
|
| 6 |
-
import
|
| 7 |
-
import
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
#
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import asyncio
|
| 5 |
+
import os
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Sequence
|
| 8 |
+
|
| 9 |
+
import deep_translator
|
| 10 |
+
import pysrt
|
| 11 |
+
import tqdm.asyncio
|
| 12 |
+
import subtitle_utils
|
| 13 |
+
from utils import format_time
|
| 14 |
+
from gemini_utils import (
|
| 15 |
+
GeminiTranslationConfig,
|
| 16 |
+
normalize_api_keys,
|
| 17 |
+
translate_with_gemini,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
# all entence endings for japanese and normal people languages
|
| 21 |
+
sentence_endings = ['.', '!', '?', ')', 'よ', 'ね',
|
| 22 |
+
'の', 'さ', 'ぞ', 'な', 'か', '!', '。', '」', '…']
|
| 23 |
+
|
| 24 |
+
# a good separator is a char or string that doenst change the translation quality but is near ever preserved in result at same or near position
|
| 25 |
+
separator = " ◌ "
|
| 26 |
+
separator_unjoin = separator.replace(' ', '')
|
| 27 |
+
chunk_max_chars = 4999
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def translate_srt_file(
|
| 31 |
+
srt_file_path: Path,
|
| 32 |
+
translated_subtitle_path: Path,
|
| 33 |
+
target_lang,
|
| 34 |
+
translate_engine: str = "google",
|
| 35 |
+
gemini_api_keys=None,
|
| 36 |
+
overwrite: bool = False,
|
| 37 |
+
):
|
| 38 |
+
"""
|
| 39 |
+
Translate SRT file using the specified engine.
|
| 40 |
+
translate_engine: "google" or "gemini"
|
| 41 |
+
gemini_api_keys: optional sequence of API keys required if translate_engine == "gemini"
|
| 42 |
+
"""
|
| 43 |
+
# Load the original SRT file
|
| 44 |
+
subs = pysrt.open(srt_file_path, encoding='utf-8')
|
| 45 |
+
|
| 46 |
+
# Extract the subtitle content and store it in a list. Also rejoin all lines splited
|
| 47 |
+
sub_content = [' '.join(sub.text.strip().splitlines()) for sub in subs]
|
| 48 |
+
|
| 49 |
+
if translate_engine == "gemini":
|
| 50 |
+
api_keys = normalize_api_keys(gemini_api_keys)
|
| 51 |
+
if not api_keys:
|
| 52 |
+
raise ValueError("Gemini API key is required for Gemini translation. Get one at https://aistudio.google.com/apikey")
|
| 53 |
+
|
| 54 |
+
# Force cleanup of previous runs to avoid resume/progress issues
|
| 55 |
+
Path(translated_subtitle_path).unlink(missing_ok=True)
|
| 56 |
+
Path(str(translated_subtitle_path) + ".progress").unlink(missing_ok=True)
|
| 57 |
+
|
| 58 |
+
subs = translate_with_gemini(
|
| 59 |
+
GeminiTranslationConfig(
|
| 60 |
+
api_keys=api_keys,
|
| 61 |
+
input_file=srt_file_path,
|
| 62 |
+
output_file=translated_subtitle_path,
|
| 63 |
+
target_language=target_lang,
|
| 64 |
+
resume=False
|
| 65 |
+
)
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
return subs
|
| 69 |
+
|
| 70 |
+
# Default: Google Translate
|
| 71 |
+
# Make chunks of at maximum $chunk_max_chars to stay under Google Translate public API limits
|
| 72 |
+
chunks = join_sentences(sub_content, chunk_max_chars) or []
|
| 73 |
+
|
| 74 |
+
# Empty list to store enumerated translated chunks
|
| 75 |
+
translated_chunks = [None] * len(chunks)
|
| 76 |
+
|
| 77 |
+
tasks = []
|
| 78 |
+
# Limit to 7 concomitant running tasks
|
| 79 |
+
semaphore = asyncio.Semaphore(7)
|
| 80 |
+
|
| 81 |
+
# Async chunks translate function
|
| 82 |
+
async def translate_async():
|
| 83 |
+
async def run_translate(index, chunk, lang):
|
| 84 |
+
while True:
|
| 85 |
+
try:
|
| 86 |
+
async with semaphore:
|
| 87 |
+
result = await asyncio.wait_for(translate_chunk(index, chunk, lang), 120)
|
| 88 |
+
translated_chunks[index] = result
|
| 89 |
+
break
|
| 90 |
+
except Exception:
|
| 91 |
+
# Restart task
|
| 92 |
+
await asyncio.sleep(3)
|
| 93 |
+
|
| 94 |
+
for index, chunk in enumerate(chunks):
|
| 95 |
+
task = asyncio.create_task(
|
| 96 |
+
run_translate(index, chunk, target_lang))
|
| 97 |
+
tasks.append(task)
|
| 98 |
+
|
| 99 |
+
for tsk in tqdm.asyncio.tqdm_asyncio.as_completed(tasks, total=len(tasks), desc="Translating", unit="chunks", unit_scale=False, leave=True, bar_format="{desc} {percentage:3.0f}% | {n_fmt}/{total_fmt} | ETA: {remaining} | ⏱: {elapsed}"):
|
| 100 |
+
await tsk
|
| 101 |
+
|
| 102 |
+
# Cria um loop de eventos e executa as tasks
|
| 103 |
+
loop = asyncio.get_event_loop()
|
| 104 |
+
loop.run_until_complete(translate_async())
|
| 105 |
+
|
| 106 |
+
print('Processing translation...', end='')
|
| 107 |
+
|
| 108 |
+
# Unjoin lines within each chunk that end with a sentence ending
|
| 109 |
+
unjoined_texts = [unjoin_sentences(
|
| 110 |
+
chunk, translated_chunks[i], separator_unjoin) or "" for i, chunk in enumerate(chunks)]
|
| 111 |
+
unjoined_texts = [text for sublist in unjoined_texts for text in sublist]
|
| 112 |
+
|
| 113 |
+
# Split lines as necessary targeting same number of lines as original string
|
| 114 |
+
for i, segment in enumerate(unjoined_texts):
|
| 115 |
+
unjoined_texts[i] = "\n".join(subtitle_utils.split_string_to_max_lines(
|
| 116 |
+
text=segment, max_width=0, max_lines=len(subs[i].text.splitlines())))
|
| 117 |
+
|
| 118 |
+
# Combine the original and translated subtitle content
|
| 119 |
+
for i, sub in enumerate(subs):
|
| 120 |
+
sub.text = unjoined_texts[i]
|
| 121 |
+
|
| 122 |
+
# Save the translated SRT file
|
| 123 |
+
os.makedirs(translated_subtitle_path.parent, exist_ok=True)
|
| 124 |
+
subs.save(translated_subtitle_path, encoding='utf-8')
|
| 125 |
+
|
| 126 |
+
print('\r ', end='\r')
|
| 127 |
+
|
| 128 |
+
return subs
|
| 129 |
+
|
| 130 |
+
# Async chunk translate function
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
async def translate_chunk(index, chunk, target_lang):
|
| 134 |
+
while True:
|
| 135 |
+
try:
|
| 136 |
+
# Translate the subtitle content of the chunk using Google Translate
|
| 137 |
+
translator = deep_translator.google.GoogleTranslator(
|
| 138 |
+
source='auto', target=target_lang)
|
| 139 |
+
translated_chunk: str = await asyncio.wait_for(asyncio.get_event_loop().run_in_executor(None, translator.translate, chunk), 30)
|
| 140 |
+
await asyncio.sleep(0)
|
| 141 |
+
|
| 142 |
+
# if nothing is retuned, return the original chunk
|
| 143 |
+
if translated_chunk is None or len(translated_chunk.replace(separator.strip(), '').split()) == 0:
|
| 144 |
+
return chunk
|
| 145 |
+
|
| 146 |
+
return translated_chunk
|
| 147 |
+
except Exception as e:
|
| 148 |
+
# If an error occurred, retry
|
| 149 |
+
del translator
|
| 150 |
+
print(
|
| 151 |
+
f"\r[chunk {index}]: Exception: {e.__doc__} Retrying in 30 seconds...", flush=True)
|
| 152 |
+
await asyncio.sleep(30)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def join_sentences(lines, max_chars):
|
| 156 |
+
"""Join sentences in chunks that stay under *max_chars* without breaking the separator mapping."""
|
| 157 |
+
joined_lines = []
|
| 158 |
+
current_chunk = ""
|
| 159 |
+
|
| 160 |
+
for index, line in enumerate(lines):
|
| 161 |
+
if not line:
|
| 162 |
+
line = '\u3164' # invisible char (not a simple space)
|
| 163 |
+
|
| 164 |
+
addition = line + separator
|
| 165 |
+
|
| 166 |
+
# if adding the current line would overflow, flush the chunk first
|
| 167 |
+
if current_chunk and len(current_chunk) + len(line) + len(separator) > max_chars:
|
| 168 |
+
joined_lines.append(current_chunk)
|
| 169 |
+
current_chunk = ""
|
| 170 |
+
|
| 171 |
+
if len(addition) > max_chars:
|
| 172 |
+
# a single line exceeds the limit; truncate conservatively
|
| 173 |
+
end_index = line.rfind(' ', 0, max_chars - (1 + len(separator)))
|
| 174 |
+
if end_index == -(1 + len(separator)):
|
| 175 |
+
end_index = max_chars - (1 + len(separator))
|
| 176 |
+
joined_lines.append((line[:end_index] + '\u2026' + separator)[:max_chars])
|
| 177 |
+
continue
|
| 178 |
+
|
| 179 |
+
current_chunk += addition
|
| 180 |
+
|
| 181 |
+
is_last_line = index == len(lines) - 1
|
| 182 |
+
ends_sentence = any(line.endswith(ending) for ending in sentence_endings)
|
| 183 |
+
|
| 184 |
+
if not ends_sentence and not is_last_line:
|
| 185 |
+
continue
|
| 186 |
+
|
| 187 |
+
if is_last_line:
|
| 188 |
+
joined_lines.append(current_chunk)
|
| 189 |
+
current_chunk = ""
|
| 190 |
+
continue
|
| 191 |
+
|
| 192 |
+
next_line = lines[index + 1] or '\u3164'
|
| 193 |
+
next_addition_length = len(next_line) + len(separator)
|
| 194 |
+
|
| 195 |
+
if len(current_chunk) + next_addition_length > max_chars:
|
| 196 |
+
joined_lines.append(current_chunk)
|
| 197 |
+
current_chunk = ""
|
| 198 |
+
|
| 199 |
+
if current_chunk:
|
| 200 |
+
joined_lines.append(current_chunk)
|
| 201 |
+
|
| 202 |
+
return joined_lines
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def unjoin_sentences(original_sentence: str, modified_sentence: str, separator: str):
|
| 206 |
+
"""
|
| 207 |
+
Splits the original and modified sentences into lines based on the separator.
|
| 208 |
+
Tries to match the number of lines between the original and modified sentences.
|
| 209 |
+
"""
|
| 210 |
+
|
| 211 |
+
if original_sentence is None:
|
| 212 |
+
return ' '
|
| 213 |
+
|
| 214 |
+
# split by separator, remove double spaces and empty or only space strings from list
|
| 215 |
+
original_lines = original_sentence.split(separator)
|
| 216 |
+
original_lines = [s.strip().replace(' ', ' ').lstrip(" ,.:;)") if s.strip().replace(' ', ' ').lstrip(" ,.:;)") else s
|
| 217 |
+
for s in original_lines if s.strip()]
|
| 218 |
+
original_lines = [s for s in original_lines if s]
|
| 219 |
+
original_lines = [s for s in original_lines if s.strip()]
|
| 220 |
+
|
| 221 |
+
if modified_sentence is None:
|
| 222 |
+
return original_lines or ' '
|
| 223 |
+
|
| 224 |
+
# fix strange formatation returned by google translate, case occuring
|
| 225 |
+
modified_sentence = modified_sentence.replace(f"{separator_unjoin} ", f"{separator_unjoin}").replace(
|
| 226 |
+
f" {separator_unjoin}", f"{separator_unjoin}").replace(
|
| 227 |
+
f"{separator_unjoin}.", f".{separator_unjoin}").replace(f"{separator_unjoin},", f",{separator_unjoin}")
|
| 228 |
+
|
| 229 |
+
# split by separator, remove double spaces and empty or only space strings from list
|
| 230 |
+
modified_lines = modified_sentence.split(separator_unjoin)
|
| 231 |
+
modified_lines = [s.strip().replace(' ', ' ').lstrip(" ,.:;)") if s.strip().replace(' ', ' ').lstrip(" ,.:;)") else s
|
| 232 |
+
for s in modified_lines if s.strip()]
|
| 233 |
+
modified_lines = [s for s in modified_lines if s]
|
| 234 |
+
modified_lines = [s for s in modified_lines if s.strip()]
|
| 235 |
+
|
| 236 |
+
# if original lines is "silence" sign, doenst translate
|
| 237 |
+
if original_lines == "..." or original_lines == "…":
|
| 238 |
+
return original_lines
|
| 239 |
+
|
| 240 |
+
# all ok, return lines
|
| 241 |
+
if len(original_lines) == len(modified_lines):
|
| 242 |
+
return modified_lines
|
| 243 |
+
|
| 244 |
+
# zero words? return original sentence, removing separator
|
| 245 |
+
original_word_count = sum(len(line.strip().split())
|
| 246 |
+
for line in original_lines)
|
| 247 |
+
modified_word_count = len(' '.join(modified_lines).strip().split())
|
| 248 |
+
if original_word_count == 0 or modified_word_count == 0:
|
| 249 |
+
return original_sentence.replace(separator, ' ').replace(' ', ' ')
|
| 250 |
+
|
| 251 |
+
# calculate proportion of words between original and translated
|
| 252 |
+
modified_words_proportion = modified_word_count / original_word_count
|
| 253 |
+
# list all modified words
|
| 254 |
+
modified_words = ' '.join(modified_lines).replace(separator, "").replace(
|
| 255 |
+
separator_unjoin, "").replace(" ", " ").strip().split(' ')
|
| 256 |
+
|
| 257 |
+
new_modified_lines = []
|
| 258 |
+
current_index = 0
|
| 259 |
+
|
| 260 |
+
# reconstruct lines based on proportion of original and translated words
|
| 261 |
+
for i in range(len(original_lines)):
|
| 262 |
+
# Calculate the number of words for the current modified sentence
|
| 263 |
+
num_words = int(
|
| 264 |
+
round(len(original_lines[i].strip().split()) * modified_words_proportion))
|
| 265 |
+
|
| 266 |
+
# Extract words from modified list
|
| 267 |
+
generated_line = ' '.join(
|
| 268 |
+
modified_words[current_index:current_index+num_words])
|
| 269 |
+
|
| 270 |
+
# Update the current index
|
| 271 |
+
current_index += num_words
|
| 272 |
+
|
| 273 |
+
# append remaining if is the last loop
|
| 274 |
+
if i == len(original_lines) - 1:
|
| 275 |
+
tail = ' '.join(modified_words[current_index:])
|
| 276 |
+
if tail:
|
| 277 |
+
generated_line = ' '.join([generated_line, tail]).strip()
|
| 278 |
+
current_index = len(modified_words)
|
| 279 |
+
|
| 280 |
+
# Add modified sentence to the new list
|
| 281 |
+
new_modified_lines.append(generated_line.replace(" ", " ").strip())
|
| 282 |
+
|
| 283 |
+
# case it continues being shorter
|
| 284 |
+
while len(new_modified_lines) < len(original_lines):
|
| 285 |
+
new_modified_lines.append(new_modified_lines[-1])
|
| 286 |
+
|
| 287 |
+
return new_modified_lines or original_lines or ' '
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def build_cli_parser() -> argparse.ArgumentParser:
|
| 291 |
+
parser = argparse.ArgumentParser(
|
| 292 |
+
prog="translate_utils",
|
| 293 |
+
description="Translate one or more SRT files using LeGen translation helpers.",
|
| 294 |
+
argument_default=argparse.SUPPRESS,
|
| 295 |
+
)
|
| 296 |
+
parser.add_argument(
|
| 297 |
+
"-i",
|
| 298 |
+
"--input_path",
|
| 299 |
+
required=True,
|
| 300 |
+
help="Path to an .srt file or a directory containing .srt files.",
|
| 301 |
+
)
|
| 302 |
+
parser.add_argument(
|
| 303 |
+
"-o",
|
| 304 |
+
"--output_path",
|
| 305 |
+
help="Destination directory or .srt file. Defaults to the source folder.",
|
| 306 |
+
)
|
| 307 |
+
parser.add_argument(
|
| 308 |
+
"--translate",
|
| 309 |
+
required=True,
|
| 310 |
+
help="Target language code (e.g., en, es, pt-BR).",
|
| 311 |
+
)
|
| 312 |
+
parser.add_argument(
|
| 313 |
+
"--translate_engine",
|
| 314 |
+
type=str.lower,
|
| 315 |
+
choices=("google", "gemini"),
|
| 316 |
+
default="google",
|
| 317 |
+
help="Translation engine to use: google (default) or gemini.",
|
| 318 |
+
)
|
| 319 |
+
parser.add_argument(
|
| 320 |
+
"--gemini_api_key",
|
| 321 |
+
action="append",
|
| 322 |
+
default=[],
|
| 323 |
+
type=str,
|
| 324 |
+
help=(
|
| 325 |
+
"Gemini API key. Repeat or separate by comma/line break to add multiple keys "
|
| 326 |
+
"(required for --translate_engine=gemini)."
|
| 327 |
+
),
|
| 328 |
+
)
|
| 329 |
+
parser.add_argument(
|
| 330 |
+
"--overwrite",
|
| 331 |
+
action="store_true",
|
| 332 |
+
help="Overwrite translated files if they already exist.",
|
| 333 |
+
)
|
| 334 |
+
return parser
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
def _output_path_is_file(candidate: Path | None) -> bool:
|
| 338 |
+
if candidate is None:
|
| 339 |
+
return False
|
| 340 |
+
if candidate.exists():
|
| 341 |
+
return candidate.is_file()
|
| 342 |
+
return candidate.suffix.lower() == ".srt"
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
def _derive_destination(source: Path, base_output: Path | None, target_language: str, input_root: Path | None = None) -> Path:
|
| 346 |
+
suffix = f"_{target_language.lower()}.srt"
|
| 347 |
+
if base_output is None:
|
| 348 |
+
return source.with_name(f"{source.stem}{suffix}")
|
| 349 |
+
|
| 350 |
+
if base_output.suffix.lower() == ".srt" and not base_output.is_dir():
|
| 351 |
+
return base_output
|
| 352 |
+
|
| 353 |
+
# If base_output is a directory (or intended to be one)
|
| 354 |
+
if input_root and source.is_relative_to(input_root):
|
| 355 |
+
rel_path = source.relative_to(input_root)
|
| 356 |
+
dest_dir = base_output / rel_path.parent
|
| 357 |
+
return dest_dir / f"{source.stem}{suffix}"
|
| 358 |
+
|
| 359 |
+
return base_output / f"{source.stem}{suffix}"
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def main(argv: Sequence[str] | None = None) -> int:
|
| 363 |
+
parser = build_cli_parser()
|
| 364 |
+
raw_argv = list(argv) if argv is not None else []
|
| 365 |
+
args = parser.parse_args(raw_argv if argv is not None else None)
|
| 366 |
+
|
| 367 |
+
input_path = Path(args.input_path).expanduser().resolve()
|
| 368 |
+
if not input_path.exists():
|
| 369 |
+
parser.error(f"Input path '{args.input_path}' does not exist.")
|
| 370 |
+
|
| 371 |
+
output_path = Path(args.output_path).expanduser().resolve() if hasattr(args, "output_path") and args.output_path else None
|
| 372 |
+
target_language = args.translate.strip()
|
| 373 |
+
if not target_language or target_language.lower() == "none":
|
| 374 |
+
parser.error("Provide a valid target language via --translate (e.g., en, es, pt-BR).")
|
| 375 |
+
|
| 376 |
+
gemini_api_keys = normalize_api_keys(getattr(args, "gemini_api_key", []))
|
| 377 |
+
translate_engine_explicit = any(str(item).startswith("--translate_engine") for item in raw_argv)
|
| 378 |
+
if (
|
| 379 |
+
not translate_engine_explicit
|
| 380 |
+
and target_language
|
| 381 |
+
and target_language.lower() != "none"
|
| 382 |
+
and args.translate_engine == "google"
|
| 383 |
+
and gemini_api_keys
|
| 384 |
+
):
|
| 385 |
+
args.translate_engine = "gemini"
|
| 386 |
+
|
| 387 |
+
if args.translate_engine == "gemini" and not gemini_api_keys:
|
| 388 |
+
parser.error("Gemini API key is required when --translate_engine=gemini.")
|
| 389 |
+
|
| 390 |
+
input_root = None
|
| 391 |
+
if input_path.is_file():
|
| 392 |
+
if input_path.suffix.lower() != ".srt":
|
| 393 |
+
parser.error("Input file must be an .srt file.")
|
| 394 |
+
source_files = [input_path]
|
| 395 |
+
elif input_path.is_dir():
|
| 396 |
+
input_root = input_path
|
| 397 |
+
source_files = sorted(input_path.rglob("*.srt"))
|
| 398 |
+
if not source_files:
|
| 399 |
+
parser.error(f"No .srt files found inside directory '{input_path}'.")
|
| 400 |
+
else:
|
| 401 |
+
parser.error(f"Input path '{input_path}' is neither a file nor a directory.")
|
| 402 |
+
|
| 403 |
+
output_is_file = _output_path_is_file(output_path)
|
| 404 |
+
if output_is_file and len(source_files) > 1:
|
| 405 |
+
parser.error("When translating multiple files the output path must be a directory.")
|
| 406 |
+
|
| 407 |
+
translated = 0
|
| 408 |
+
skipped = 0
|
| 409 |
+
target_suffix = f"_{target_language.lower()}.srt"
|
| 410 |
+
for source in source_files:
|
| 411 |
+
if source.name.lower().endswith(target_suffix):
|
| 412 |
+
skipped += 1
|
| 413 |
+
continue
|
| 414 |
+
|
| 415 |
+
destination = _derive_destination(source, output_path, target_language, input_root)
|
| 416 |
+
if destination.exists() and not getattr(args, "overwrite", False):
|
| 417 |
+
print(f"Skipping existing file {destination}")
|
| 418 |
+
skipped += 1
|
| 419 |
+
continue
|
| 420 |
+
|
| 421 |
+
translate_srt_file(
|
| 422 |
+
source,
|
| 423 |
+
destination,
|
| 424 |
+
target_language,
|
| 425 |
+
translate_engine=args.translate_engine,
|
| 426 |
+
gemini_api_keys=gemini_api_keys,
|
| 427 |
+
overwrite=getattr(args, "overwrite", False),
|
| 428 |
+
)
|
| 429 |
+
print(f"Translated {source} -> {destination}")
|
| 430 |
+
translated += 1
|
| 431 |
+
|
| 432 |
+
total = len(source_files)
|
| 433 |
+
print(f"Finished translating {translated}/{total} file(s). {skipped} skipped.")
|
| 434 |
+
return 0
|
| 435 |
+
|
| 436 |
+
|
| 437 |
+
if __name__ == "__main__": # pragma: no cover
|
| 438 |
+
raise SystemExit(main())
|
utils.py
CHANGED
|
@@ -1,72 +1,102 @@
|
|
| 1 |
-
import time
|
| 2 |
-
from pathlib import Path
|
| 3 |
-
|
| 4 |
-
from contextlib import contextmanager
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import time
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
from contextlib import contextmanager
|
| 5 |
+
import re
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
@contextmanager
|
| 9 |
+
def time_task(message_start=None, end=' ', message="⏱ Took"):
|
| 10 |
+
if message_start:
|
| 11 |
+
print(message_start, end=end, flush=True)
|
| 12 |
+
start_time = time.time()
|
| 13 |
+
yield
|
| 14 |
+
end_time = time.time()
|
| 15 |
+
elapsed_time = end_time - start_time
|
| 16 |
+
formatted_elapsed_time = format_time(elapsed_time)
|
| 17 |
+
print(f"{message} {formatted_elapsed_time}", flush=True)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def format_time(elapsed_time):
|
| 21 |
+
hours, rem = divmod(elapsed_time, 3600)
|
| 22 |
+
minutes, seconds = divmod(rem, 60)
|
| 23 |
+
|
| 24 |
+
# Check and format the non-zero time units
|
| 25 |
+
parts = []
|
| 26 |
+
if hours:
|
| 27 |
+
parts.append(f"{int(hours)}h")
|
| 28 |
+
if minutes:
|
| 29 |
+
parts.append(f"{int(minutes)}m")
|
| 30 |
+
if seconds or not parts:
|
| 31 |
+
# Add seconds if it's the only non-zero unit or if all units are zero
|
| 32 |
+
parts.append(f"{int(seconds)}s")
|
| 33 |
+
|
| 34 |
+
return ' '.join(parts)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def time_func(func):
|
| 38 |
+
def wrapper(*args, **kwargs):
|
| 39 |
+
start_time = time.time()
|
| 40 |
+
result = func(*args, **kwargs)
|
| 41 |
+
end_time = time.time()
|
| 42 |
+
elapsed_time = end_time - start_time
|
| 43 |
+
formatted_elapsed_time = format_time(elapsed_time)
|
| 44 |
+
print(f"Execution time of {func.__name__}: {formatted_elapsed_time}")
|
| 45 |
+
return result
|
| 46 |
+
return wrapper
|
| 47 |
+
|
| 48 |
+
def check_other_extensions(file_path, extensions_to_check):
|
| 49 |
+
"""
|
| 50 |
+
Check the existence of files with the same name but different extensions
|
| 51 |
+
in the same folder.
|
| 52 |
+
|
| 53 |
+
Parameters:
|
| 54 |
+
- file_path (str): The path of the file to check.
|
| 55 |
+
- extensions_to_check (list): List of extensions to check.
|
| 56 |
+
|
| 57 |
+
Returns:
|
| 58 |
+
- list: List of existing file paths with different extensions.
|
| 59 |
+
"""
|
| 60 |
+
file_path = Path(file_path)
|
| 61 |
+
folder = file_path.parent
|
| 62 |
+
base_name = file_path.stem
|
| 63 |
+
|
| 64 |
+
matching_files = [
|
| 65 |
+
folder / (base_name + ext)
|
| 66 |
+
for ext in extensions_to_check
|
| 67 |
+
if (folder / (base_name + ext)).exists()
|
| 68 |
+
]
|
| 69 |
+
|
| 70 |
+
return matching_files
|
| 71 |
+
|
| 72 |
+
video_extensions = {".mp4", ".webm", ".mkv", ".avi", ".mov", ".wmv", ".flv", ".vob", ".mts", ".m2ts", ".ts", ".yuv", ".mpg", ".mp2", ".mpeg", ".mpe", ".mpv", ".m2v", ".m4v", ".3gp", ".3g2", ".nsv", ".mts"}
|
| 73 |
+
audio_extensions = {".aa", ".aac", ".aax", ".act", ".aiff", ".alac", ".amr", ".ape", ".au", ".awb", ".dss", ".dvf", ".flac", ".gsm", ".iklax", ".ivs", ".m4a", ".m4b", ".m4p", ".mpga", ".mmf", ".mp3", ".mpc", ".msv", ".nmf", ".ogg", ".oga", ".mogg", ".opus", ".ra", ".rm", ".raw", ".rf64", ".sln", ".tta", ".voc", ".vox", ".wav", ".wma", ".wv", ".webm", ".8svx"}
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
_LANG_SUFFIX_RE = re.compile(r"^(?P<lang>[A-Za-z]{2,3}(?:-[A-Za-z]{2,4})?)$")
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def split_lang_suffix(stem: str) -> tuple[str, str | None]:
|
| 80 |
+
"""Split a trailing language suffix from a filename stem.
|
| 81 |
+
|
| 82 |
+
Examples:
|
| 83 |
+
- "video_pt-br" -> ("video", "pt-br")
|
| 84 |
+
- "video_en" -> ("video", "en")
|
| 85 |
+
- "video" -> ("video", None)
|
| 86 |
+
|
| 87 |
+
The matching is intentionally conservative to avoid misclassifying stems.
|
| 88 |
+
"""
|
| 89 |
+
if not stem:
|
| 90 |
+
return stem, None
|
| 91 |
+
|
| 92 |
+
if "_" not in stem:
|
| 93 |
+
return stem, None
|
| 94 |
+
|
| 95 |
+
base, maybe_lang = stem.rsplit("_", 1)
|
| 96 |
+
if not base:
|
| 97 |
+
return stem, None
|
| 98 |
+
|
| 99 |
+
if _LANG_SUFFIX_RE.match(maybe_lang or ""):
|
| 100 |
+
return base, maybe_lang.lower()
|
| 101 |
+
|
| 102 |
+
return stem, None
|
uv.lock
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
whisper_utils.py
CHANGED
|
@@ -1,50 +1,61 @@
|
|
| 1 |
-
import os
|
| 2 |
-
from pathlib import Path
|
| 3 |
-
|
| 4 |
-
import pysrt
|
| 5 |
-
import whisper
|
| 6 |
-
import whisper.transcribe
|
| 7 |
-
import whisperx
|
| 8 |
-
import
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
import pysrt
|
| 5 |
+
import whisper
|
| 6 |
+
import whisper.transcribe
|
| 7 |
+
import whisperx_legen_fork as whisperx
|
| 8 |
+
from whisperx_legen_fork import alignment, asr, utils, audio
|
| 9 |
+
import subtitle_utils
|
| 10 |
+
from utils import time_task
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def transcribe_audio(model: whisper.model, audio_path: Path, srt_path: Path, lang: str = None, disable_fp16: bool = False):
|
| 14 |
+
# Load audio
|
| 15 |
+
audio = whisper.load_audio(file=audio_path.as_posix())
|
| 16 |
+
|
| 17 |
+
# Transcribe
|
| 18 |
+
with time_task():
|
| 19 |
+
transcribe = model.transcribe(audio=audio, language=lang, fp16=False if disable_fp16 else True, verbose=False)
|
| 20 |
+
|
| 21 |
+
# Align if possible
|
| 22 |
+
if lang in alignment.DEFAULT_ALIGN_MODELS_HF or lang in alignment.DEFAULT_ALIGN_MODELS_TORCH:
|
| 23 |
+
model_device = str(getattr(model, "device", "cpu"))
|
| 24 |
+
alignment_device = "cuda" if model_device.startswith("cuda") else "cpu"
|
| 25 |
+
if alignment_device == "cuda":
|
| 26 |
+
try:
|
| 27 |
+
import torch
|
| 28 |
+
if not torch.cuda.is_available():
|
| 29 |
+
alignment_device = "cpu"
|
| 30 |
+
except Exception:
|
| 31 |
+
alignment_device = "cpu"
|
| 32 |
+
with time_task(message_start="Running alignment..."):
|
| 33 |
+
try:
|
| 34 |
+
model_a, metadata = alignment.load_align_model(language_code=lang, device=alignment_device)
|
| 35 |
+
transcribe = alignment.align(transcript=transcribe["segments"], model=model_a, align_model_metadata=metadata, audio=audio, device=alignment_device, return_char_alignments=True)
|
| 36 |
+
except Exception:
|
| 37 |
+
model_a, metadata = alignment.load_align_model(language_code=lang, device="cpu") # force load on cpu due errors on gpu
|
| 38 |
+
transcribe = alignment.align(transcript=transcribe["segments"], model=model_a, align_model_metadata=metadata, audio=audio, device="cpu", return_char_alignments=True)
|
| 39 |
+
else:
|
| 40 |
+
print(f"Language {lang} not suported for alignment. Skipping this step")
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
# Format subtitles
|
| 44 |
+
segments = subtitle_utils.format_segments(transcribe['segments'])
|
| 45 |
+
|
| 46 |
+
# Save the subtitle file
|
| 47 |
+
subtitle_utils.SaveSegmentsToSrt(segments, srt_path)
|
| 48 |
+
|
| 49 |
+
return transcribe
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def detect_language(model: str, audio_path: Path):
|
| 53 |
+
# load audio and pad/trim it to fit 30 seconds
|
| 54 |
+
audio = whisper.load_audio(audio_path.as_posix())
|
| 55 |
+
audio = whisper.pad_or_trim(audio)
|
| 56 |
+
# make log-Mel spectrogram and move to the same device as the model
|
| 57 |
+
mel = whisper.log_mel_spectrogram(audio).to(model.device)
|
| 58 |
+
|
| 59 |
+
# detect the spoken language
|
| 60 |
+
_, probs = model.detect_language(mel)
|
| 61 |
+
return max(probs, key=probs.get)
|
whisperx_utils.py
CHANGED
|
@@ -1,79 +1,87 @@
|
|
| 1 |
-
import os
|
| 2 |
-
from pathlib import Path
|
| 3 |
-
|
| 4 |
-
import whisperx
|
| 5 |
-
import whisper
|
| 6 |
-
|
| 7 |
-
import whisper_utils
|
| 8 |
-
import subtitle_utils
|
| 9 |
-
from utils import time_task
|
| 10 |
-
|
| 11 |
-
def transcribe_audio(model:
|
| 12 |
-
audio =
|
| 13 |
-
|
| 14 |
-
# Define the progress callback function
|
| 15 |
-
def progress_callback(state, current: int = None, total: int = None):
|
| 16 |
-
args = state, current, total
|
| 17 |
-
args = [arg for arg in args if arg is not None]
|
| 18 |
-
|
| 19 |
-
if len(args) == 1:
|
| 20 |
-
state = args[0]
|
| 21 |
-
if len(args) > 1:
|
| 22 |
-
total = args[-1]
|
| 23 |
-
current = args[-2]
|
| 24 |
-
state = None
|
| 25 |
-
if len(args) > 2:
|
| 26 |
-
state = args[-3]
|
| 27 |
-
|
| 28 |
-
try:
|
| 29 |
-
if state is None:
|
| 30 |
-
state = "WhisperX"
|
| 31 |
-
elif type(state) == 'String' or type(state) == int:
|
| 32 |
-
state = state
|
| 33 |
-
else:
|
| 34 |
-
state = state.value
|
| 35 |
-
except:
|
| 36 |
-
state = "WhisperX"
|
| 37 |
-
|
| 38 |
-
print('\r \r' + state + ((': ' + str(round(current/total*100)) + '%') if current and total else '') + ((' [' + str(current) + '/' + str(total) + ']') if current and total else ''), end=' ', flush=True)
|
| 39 |
-
|
| 40 |
-
# Transcribe
|
| 41 |
-
with time_task("Running WhisperX transcription engine...", end='\n'):
|
| 42 |
-
transcribe = model.transcribe(audio=audio, language=lang, batch_size=batch_size, on_progress=progress_callback)
|
| 43 |
-
|
| 44 |
-
# Align if possible
|
| 45 |
-
if lang in
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
import whisperx_legen_fork as whisperx
|
| 5 |
+
import whisper
|
| 6 |
+
from whisperx_legen_fork import asr, audio as wx_audio, alignment, utils
|
| 7 |
+
import whisper_utils
|
| 8 |
+
import subtitle_utils
|
| 9 |
+
from utils import time_task
|
| 10 |
+
|
| 11 |
+
def transcribe_audio(model: asr.WhisperModel, audio_path: Path, srt_path: Path, lang: str = None, device: str = "cpu", batch_size: int = 4):
|
| 12 |
+
audio = wx_audio.load_audio(file=audio_path.as_posix(), sr=model.model.feature_extractor.sampling_rate)
|
| 13 |
+
|
| 14 |
+
# Define the progress callback function
|
| 15 |
+
def progress_callback(state, current: int = None, total: int = None):
|
| 16 |
+
args = state, current, total
|
| 17 |
+
args = [arg for arg in args if arg is not None]
|
| 18 |
+
|
| 19 |
+
if len(args) == 1:
|
| 20 |
+
state = args[0]
|
| 21 |
+
if len(args) > 1:
|
| 22 |
+
total = args[-1]
|
| 23 |
+
current = args[-2]
|
| 24 |
+
state = None
|
| 25 |
+
if len(args) > 2:
|
| 26 |
+
state = args[-3]
|
| 27 |
+
|
| 28 |
+
try:
|
| 29 |
+
if state is None:
|
| 30 |
+
state = "WhisperX"
|
| 31 |
+
elif type(state) == 'String' or type(state) == int:
|
| 32 |
+
state = state
|
| 33 |
+
else:
|
| 34 |
+
state = state.value
|
| 35 |
+
except:
|
| 36 |
+
state = "WhisperX"
|
| 37 |
+
|
| 38 |
+
print('\r \r' + state + ((': ' + str(round(current/total*100)) + '%') if current and total else '') + ((' [' + str(current) + '/' + str(total) + ']') if current and total else ''), end=' ', flush=True)
|
| 39 |
+
|
| 40 |
+
# Transcribe
|
| 41 |
+
with time_task("Running WhisperX transcription engine...", end='\n'):
|
| 42 |
+
transcribe = model.transcribe(audio=audio, language=lang, batch_size=batch_size, on_progress=progress_callback)
|
| 43 |
+
|
| 44 |
+
# Align if possible
|
| 45 |
+
if lang in alignment.DEFAULT_ALIGN_MODELS_HF or lang in alignment.DEFAULT_ALIGN_MODELS_TORCH:
|
| 46 |
+
alignment_device = "cuda" if str(device).startswith("cuda") else "cpu"
|
| 47 |
+
if alignment_device == "cuda":
|
| 48 |
+
try:
|
| 49 |
+
import torch
|
| 50 |
+
if not torch.cuda.is_available():
|
| 51 |
+
alignment_device = "cpu"
|
| 52 |
+
except Exception:
|
| 53 |
+
alignment_device = "cpu"
|
| 54 |
+
with time_task(message_start="Running alignment...", end='\n'):
|
| 55 |
+
try:
|
| 56 |
+
model_a, metadata = alignment.load_align_model(language_code=lang, device=alignment_device)
|
| 57 |
+
transcribe = alignment.align(transcript=transcribe["segments"], model=model_a, align_model_metadata=metadata, audio=audio, device=alignment_device, return_char_alignments=True, on_progress=progress_callback)
|
| 58 |
+
except Exception:
|
| 59 |
+
model_a, metadata = alignment.load_align_model(language_code=lang, device="cpu") # force load on cpu due errors on gpu
|
| 60 |
+
transcribe = alignment.align(transcript=transcribe["segments"], model=model_a, align_model_metadata=metadata, audio=audio, device="cpu", return_char_alignments=True, on_progress=progress_callback)
|
| 61 |
+
else:
|
| 62 |
+
print(f"Language {lang} not suported for alignment. Skipping this step")
|
| 63 |
+
|
| 64 |
+
# Format subtitles
|
| 65 |
+
segments = subtitle_utils.format_segments(transcribe['segments'])
|
| 66 |
+
|
| 67 |
+
# Save the subtitle file
|
| 68 |
+
subtitle_utils.SaveSegmentsToSrt(segments, srt_path)
|
| 69 |
+
|
| 70 |
+
return transcribe
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def detect_language(model: asr.WhisperModel, audio_path: Path):
|
| 74 |
+
try:
|
| 75 |
+
if os.getenv("COLAB_RELEASE_TAG"):
|
| 76 |
+
raise Exception("Method invalid for Google Colab")
|
| 77 |
+
audio = wx_audio.load_audio(audio_path.as_posix(), model.model.feature_extractor.sampling_rate)
|
| 78 |
+
audio = wx_audio.pad_or_trim(audio, model.model.feature_extractor.n_samples)
|
| 79 |
+
mel = wx_audio.log_mel_spectrogram(audio, n_mels=model.model.model.n_mels)
|
| 80 |
+
encoder_output = model.model.encode(mel)
|
| 81 |
+
results = model.model.model.detect_language(encoder_output)
|
| 82 |
+
language_token, language_probability = results[0][0]
|
| 83 |
+
return language_token[2:-2]
|
| 84 |
+
except:
|
| 85 |
+
print("using whisper base model for detection: ", end='')
|
| 86 |
+
whisper_model = whisper.load_model("base", device="cpu", in_memory=True)
|
| 87 |
+
return whisper_utils.detect_language(model=whisper_model, audio_path=audio_path)
|