Upload folder using huggingface_hub

.gitignore

@@ -103,6 +103,7 @@ test.py
 /models/
 /scripts/
 
+
 /src/tools/
 /src/utils/
 /src/visualization/

app.py

@@ -11,15 +11,12 @@ from src.models.taskers.inferencer import infer
 
 logger = get_logger("Application", is_stream=True)
 
-def setup_environment():
-    cmd = ['bash', 'scripts/prepare.sh', '--platform', 'gradio']
-    subprocess.run(cmd, shell=False, capture_output=False, stdout=None)
-
-setup_environment()
+cmd = ['bash', 'src/prepare.sh', '--platform', 'gradio']
+subprocess.run(cmd, shell=False, capture_output=False, stdout=None)
 logger.info("Environment is set up")
 
 model, cfg, saved_cfg, llm_tokenizer = load_ensemble_model(
-    os.path.join(repo_dir, 'vavsp_llm.yaml')
+    os.path.join(repo_dir, 'src/models/vavsp_llm.yaml')
 )
 logger.info("Loaded model")
 
@@ -64,7 +61,7 @@ if __name__ == "__main__":
                 format='mp4'
             ),
             gr.Slider(
-                minimum=1, maximum=999,
+                minimum=1, maximum=200,
                 value=3, step=1,
                 label='Second',
             ),
@@ -74,4 +71,4 @@ if __name__ == "__main__":
         description="Vietnamese Automatic Speech Recognition Utilizing Audio and Visual Data"
     )
 
-    app.queue().launch(share=True)
+    app.launch(share=True)

src/models/taskers/embedder.py

@@ -61,7 +61,7 @@ class Embedder(Tasker):
             'ffmpeg', '-y',
             '-loglevel', 'panic',
             '-i', video_path,
-            '-vf', f'subtitles={subtitle_path}:force_style="PrimaryColour=&HFFFFFF,BorderStyle=4",BackColour=0',
+            '-vf', f'\"subtitles={subtitle_path}:force_style=\'PrimaryColour=&HFFFFFF,BorderStyle=4,BackColour=0\'\"',
             output_path,
         ]
 

src/models/vavsp_llm.yaml

@@ -0,0 +1,197 @@
+_name: null
+task: null
+generation:
+  _name: null
+  beam: 20
+  nbest: 1
+  max_len_a: 1.0
+  max_len_b: 0
+  min_len: 1
+  match_source_len: false
+  unnormalized: false
+  no_early_stop: false
+  no_beamable_mm: false
+  lenpen: 0.0
+  unkpen: 0.0
+  replace_unk: null
+  sacrebleu: false
+  score_reference: false
+  prefix_size: 0
+  no_repeat_ngram_size: 0
+  sampling: false
+  sampling_topk: -1
+  sampling_topp: -1.0
+  constraints: null
+  temperature: 1.0
+  diverse_beam_groups: -1
+  diverse_beam_strength: 0.5
+  diversity_rate: -1.0
+  print_alignment: null
+  print_step: false
+  lm_path: null
+  lm_weight: 0.0
+  iter_decode_eos_penalty: 0.0
+  iter_decode_max_iter: 10
+  iter_decode_force_max_iter: false
+  iter_decode_with_beam: 1
+  iter_decode_with_external_reranker: false
+  retain_iter_history: false
+  retain_dropout: false
+  retain_dropout_modules: null
+  decoding_format: null
+  no_seed_provided: false
+common:
+  _name: null
+  no_progress_bar: false
+  log_interval: 100
+  log_format: null
+  log_file: null
+  tensorboard_logdir: null
+  wandb_project: null
+  azureml_logging: false
+  seed: 1
+  cpu: false
+  tpu: false
+  bf16: false
+  memory_efficient_bf16: false
+  fp16: false
+  memory_efficient_fp16: false
+  fp16_no_flatten_grads: false
+  fp16_init_scale: 128
+  fp16_scale_window: null
+  fp16_scale_tolerance: 0.0
+  on_cpu_convert_precision: false
+  min_loss_scale: 0.0001
+  threshold_loss_scale: null
+  amp: false
+  amp_batch_retries: 2
+  amp_init_scale: 128
+  amp_scale_window: null
+  user_dir: src
+  empty_cache_freq: 0
+  all_gather_list_size: 16384
+  model_parallel_size: 1
+  quantization_config_path: null
+  profile: false
+  reset_logging: false
+  suppress_crashes: false
+  use_plasma_view: false
+  plasma_path: /tmp/plasma
+common_eval:
+  _name: null
+  path: src/models/checkpoints/checkpoint_best.pt
+  post_process: null
+  quiet: false
+  model_overrides: '{}'
+  results_path: decode/vsr/vi
+checkpoint:
+  _name: null
+  save_dir: checkpoints
+  restore_file: checkpoint_last.pt
+  finetune_from_model: null
+  reset_dataloader: false
+  reset_lr_scheduler: false
+  reset_meters: false
+  reset_optimizer: false
+  optimizer_overrides: '{}'
+  save_interval: 1
+  save_interval_updates: 0
+  keep_interval_updates: -1
+  keep_interval_updates_pattern: -1
+  keep_last_epochs: -1
+  keep_best_checkpoints: -1
+  no_save: false
+  no_epoch_checkpoints: false
+  no_last_checkpoints: false
+  no_save_optimizer_state: false
+  best_checkpoint_metric: loss
+  maximize_best_checkpoint_metric: false
+  patience: -1
+  checkpoint_suffix: ''
+  checkpoint_shard_count: 1
+  load_checkpoint_on_all_dp_ranks: false
+  write_checkpoints_asynchronously: false
+  model_parallel_size: 1
+distributed_training:
+  _name: null
+  distributed_world_size: 1
+  distributed_num_procs: 1
+  distributed_rank: 0
+  distributed_backend: nccl
+  distributed_init_method: null
+  distributed_port: -1
+  device_id: 0
+  distributed_no_spawn: false
+  ddp_backend: pytorch_ddp
+  ddp_comm_hook: none
+  bucket_cap_mb: 25
+  fix_batches_to_gpus: false
+  find_unused_parameters: false
+  fast_stat_sync: false
+  heartbeat_timeout: -1
+  broadcast_buffers: false
+  slowmo_momentum: null
+  slowmo_algorithm: LocalSGD
+  localsgd_frequency: 3
+  nprocs_per_node: 1
+  pipeline_model_parallel: false
+  pipeline_balance: null
+  pipeline_devices: null
+  pipeline_chunks: 0
+  pipeline_encoder_balance: null
+  pipeline_encoder_devices: null
+  pipeline_decoder_balance: null
+  pipeline_decoder_devices: null
+  pipeline_checkpoint: never
+  zero_sharding: none
+  fp16: false
+  memory_efficient_fp16: false
+  tpu: false
+  no_reshard_after_forward: false
+  fp32_reduce_scatter: false
+  cpu_offload: false
+  use_sharded_state: false
+dataset:
+  _name: null
+  num_workers: 0
+  skip_invalid_size_inputs_valid_test: false
+  max_tokens: 3000
+  batch_size: null
+  required_batch_size_multiple: 8
+  required_seq_len_multiple: 1
+  dataset_impl: null
+  data_buffer_size: 10
+  train_subset: train
+  valid_subset: valid
+  combine_valid_subsets: null
+  ignore_unused_valid_subsets: false
+  validate_interval: 1
+  validate_interval_updates: 0
+  validate_after_updates: 0
+  fixed_validation_seed: null
+  disable_validation: false
+  max_tokens_valid: 3000
+  batch_size_valid: null
+  max_valid_steps: null
+  curriculum: 0
+  gen_subset: test
+  num_shards: 1
+  shard_id: 0
+override:
+  _name: null
+  noise_wav: null
+  noise_prob: 0.0
+  noise_snr: 0.0
+  modalities:
+  - visual
+  - audio
+  data: src/models/dataset/vsr/vi
+  label_dir: src/models/dataset/vsr/vi
+  labels:
+  - km
+  label_rate: -1
+  eval_bleu: false
+  llm_ckpt_path: vilm/vinallama-2.7b
+  w2v_path: src/models/checkpoints/large_vox_iter5.pt
+  demo: false
+is_ax: false

src/prepare.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+usage() {
+  echo "Description:  Prepare environment when running on cloud platform kaggle or colab
+Usage:
+  $0
+    --platform <platform>         Cloud platform where to run project.
+                                  2 available platforms 'kaggle' and 'any'. Default: kaggle
+  "
+  exit 1
+}
+
+platform="any"
+
+while [[ "$#" -gt 0 ]]; do
+  case "$1" in
+    --platform)
+      if [[ -z "$2" ]];
+      then
+        echo "MISSING platform value"
+        usage
+      fi
+      platform="$2"
+      shift 2
+    ;;
+  esac
+done
+
+
+# Download and set up miniconda
+MINICONDA_INSTALLER_SCRIPT=Miniconda3-py310_23.11.0-2-Linux-x86_64.sh
+MINICONDA_PREFIX=/usr/local
+wget https://repo.continuum.io/miniconda/$MINICONDA_INSTALLER_SCRIPT
+chmod +x $MINICONDA_INSTALLER_SCRIPT
+./$MINICONDA_INSTALLER_SCRIPT -b -f -p $MINICONDA_PREFIX
+
+# Update version python
+conda install --channel defaults conda python=3.10 --yes
+conda update --channel defaults --all --yes
+
+# Install ffmpeg to process media file
+conda install -c conda-forge ffmpeg==7.0.1 --yes
+
+# Require git to install packages from github
+if [[ "$platform" == 'kaggle' ]]
+then
+  conda install -c conda-forge git --yes
+fi
+
+# Install hf-transfer
+pip install hf-transfer
+env HF_HUB_ENABLE_HF_TRANSFER=1