Upload my pretrained model

STA-V2A/best/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5ed7e5684512bddd3071d4bac9aa772f2c328b46a969ee92d01b2b10ec599aa
+size 2193962117

STA-V2A/best/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c068ef545f11557271908557c3edf0dba6dfe87ddb37b7f5014f8bf87c43cfa
+size 442714114

STA-V2A/best/pytorch_model_1.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba6d10e4a565f922395be2e8a9e872a5342005055ebaf64963d195553e05ae71
+size 8538252

STA-V2A/best/pytorch_model_2.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94e6d440140258253b183dfee25bafacfb2712f58638680acc9f3730059cd9ce
+size 3808963356

STA-V2A/best/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1adb68ee3baf1a31d35e3864013abca08ad8f2ff5cf272968fef97fcfda34db0
+size 15580

STA-V2A/best/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd73e2d56fec237d53bae1e2e58c3988365819ae34f2d633cd5944c172ab998b
+size 1000

STA-V2A/summary.jsonl

@@ -0,0 +1 @@
+{"train_file": "", "validation_file": "", "test_file": "", "num_examples": -1, "text_encoder_name": "./mt5-large", "scheduler_name": "stabilityai/stable-diffusion-2-1", "unet_model_name": "./saved/unet/pretrain_ft", "unet_model_config": null, "hf_model": null, "snr_gamma": 5.0, "freeze_text_encoder": true, "text_column": "caption", "audio_column": "audio_location", "video_column": "frame_pt", "feature_column": "cavp_feature_location", "augment": false, "uncondition": true, "prefix": null, "per_device_train_batch_size": 20, "per_device_eval_batch_size": 20, "learning_rate": 3e-05, "weight_decay": 1e-08, "num_train_epochs": 40, "max_train_steps": null, "gradient_accumulation_steps": 2, "lr_scheduler_type": "linear", "num_warmup_steps": 0, "adam_beta1": 0.9, "adam_beta2": 0.999, "adam_weight_decay": 0.01, "adam_epsilon": 1e-08, "output_dir": "saved/STA-V2A", "seed": null, "checkpointing_steps": "best", "save_every": 1, "resume_from_checkpoint": null, "resume_diff_from_checkpoint": null, "vae_model": "audioldm-s-full", "sample_rate": 16000, "with_tracking": false, "var_len": false, "report_to": "all", "video_fps": 40, "fraze_unet": false, "predict_onset_model": null, "has_global_video_feature": true, "use_feature_window": true, "guidance_free_rate": 0.2, "Onset_weight": 1.0}

pretrained_model/audioldm-s-full.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f8d1923410622be823279b61967d27a2df3fd03ddd764afb298e7c20ef8877d
+size 2558947469

pretrained_model/clap/clap-htsat-unfused/README.md

@@ -0,0 +1,107 @@
+---
+license: apache-2.0
+---
+# Model card for CLAP
+
+Model card for CLAP: Contrastive Language-Audio Pretraining
+
+![clap_image](https://s3.amazonaws.com/moonup/production/uploads/1678811100805-62441d1d9fdefb55a0b7d12c.png)
+
+
+#  Table of Contents
+
+0. [TL;DR](#TL;DR)
+1. [Model Details](#model-details)
+2. [Usage](#usage)
+3. [Uses](#uses)
+4. [Citation](#citation)
+
+# TL;DR
+
+The abstract of the paper states that: 
+
+> Contrastive learning has shown remarkable success in the field of multimodal representation learning. In this paper, we propose a pipeline of contrastive language-audio pretraining to develop an audio representation by combining audio data with natural language descriptions. To accomplish this target, we first release LAION-Audio-630K, a large collection of 633,526 audio-text pairs from different data sources. Second, we construct a contrastive language-audio pretraining model by considering different audio encoders and text encoders. We incorporate the feature fusion mechanism and keyword-to-caption augmentation into the model design to further enable the model to process audio inputs of variable lengths and enhance the performance. Third, we perform comprehensive experiments to evaluate our model across three tasks: text-to-audio retrieval, zero-shot audio classification, and supervised audio classification. The results demonstrate that our model achieves superior performance in text-to-audio retrieval task. In audio classification tasks, the model achieves state-of-the-art performance in the zero-shot setting and is able to obtain performance comparable to models' results in the non-zero-shot setting. LAION-Audio-630K and the proposed model are both available to the public.
+
+
+# Usage
+
+You can use this model for zero shot audio classification or extracting audio and/or textual features.
+
+# Uses
+
+## Perform zero-shot audio classification
+
+### Using `pipeline`
+
+```python
+from datasets import load_dataset
+from transformers import pipeline
+
+dataset = load_dataset("ashraq/esc50")
+audio = dataset["train"]["audio"][-1]["array"]
+
+audio_classifier = pipeline(task="zero-shot-audio-classification", model="laion/clap-htsat-unfused")
+output = audio_classifier(audio, candidate_labels=["Sound of a dog", "Sound of vaccum cleaner"])
+print(output)
+>>> [{"score": 0.999, "label": "Sound of a dog"}, {"score": 0.001, "label": "Sound of vaccum cleaner"}]
+```
+
+## Run the model:
+
+You can also get the audio and text embeddings using `ClapModel`
+
+### Run the model on CPU:
+
+```python
+from datasets import load_dataset
+from transformers import ClapModel, ClapProcessor
+
+librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+audio_sample = librispeech_dummy[0]
+
+model = ClapModel.from_pretrained("laion/clap-htsat-unfused")
+processor = ClapProcessor.from_pretrained("laion/clap-htsat-unfused")
+
+inputs = processor(audios=audio_sample["audio"]["array"], return_tensors="pt")
+audio_embed = model.get_audio_features(**inputs)
+```
+
+### Run the model on GPU:
+
+```python
+from datasets import load_dataset
+from transformers import ClapModel, ClapProcessor
+
+librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+audio_sample = librispeech_dummy[0]
+
+model = ClapModel.from_pretrained("laion/clap-htsat-unfused").to(0)
+processor = ClapProcessor.from_pretrained("laion/clap-htsat-unfused")
+
+inputs = processor(audios=audio_sample["audio"]["array"], return_tensors="pt").to(0)
+audio_embed = model.get_audio_features(**inputs)
+```
+
+
+# Citation
+
+If you are using this model for your work, please consider citing the original paper:
+```
+@misc{https://doi.org/10.48550/arxiv.2211.06687,
+  doi = {10.48550/ARXIV.2211.06687},
+  
+  url = {https://arxiv.org/abs/2211.06687},
+  
+  author = {Wu, Yusong and Chen, Ke and Zhang, Tianyu and Hui, Yuchen and Berg-Kirkpatrick, Taylor and Dubnov, Shlomo},
+  
+  keywords = {Sound (cs.SD), Audio and Speech Processing (eess.AS), FOS: Computer and information sciences, FOS: Computer and information sciences, FOS: Electrical engineering, electronic engineering, information engineering, FOS: Electrical engineering, electronic engineering, information engineering},
+  
+  title = {Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation},
+  
+  publisher = {arXiv},
+  
+  year = {2022},
+  
+  copyright = {Creative Commons Attribution 4.0 International}
+}
+```

pretrained_model/clap/laion_clap/README.md

@@ -0,0 +1,3 @@
+---
+license: cc0-1.0
+---

pretrained_model/clap/msclap/README.md

@@ -0,0 +1,110 @@
+---
+license: ms-pl
+tags:
+  - contrastive audio language pretraining
+  - audio
+  - music
+  - emotion
+  - sound events
+  - bioacoustics
+  - retrieval
+  - captioning
+  - zero-shot
+  - audio-text
+  - CLAP
+---
+
+###### [Overview](#CLAP) | [Setup](#Setup) | [CLAP weights](#CLAP-weights) | [Usage](#Usage) | [Examples](#Examples) | [Citation](#Citation)
+
+# CLAP
+
+CLAP (Contrastive Language-Audio Pretraining) is a model that learns acoustic concepts from natural language supervision and enables “Zero-Shot” inference. The model has been extensively evaluated in 26 audio downstream tasks achieving SoTA in several of them including classification, retrieval, and captioning.
+
+<img width="832" alt="clap_diagrams" src="docs/clap2_diagram.png">
+
+## Setup
+
+First, install python 3.8 or higher (3.11 recommended). Then, install CLAP using either of the following:
+
+```shell
+# Install pypi pacakge
+pip install msclap
+
+# Or Install latest (unstable) git source
+pip install git+https://github.com/microsoft/CLAP.git
+```
+
+## NEW CLAP weights
+CLAP weights: versions _2022_, _2023_, and _clapcap_
+
+_clapcap_ is the audio captioning model that uses the 2023 encoders.
+
+## Usage
+
+CLAP code is in https://github.com/microsoft/CLAP
+
+- Zero-Shot Classification and Retrieval
+```python
+from msclap import CLAP
+
+# Load model (Choose between versions '2022' or '2023')
+clap_model = CLAP("<PATH TO WEIGHTS>", version = '2023', use_cuda=False)
+
+# Extract text embeddings
+text_embeddings = clap_model.get_text_embeddings(class_labels: List[str])
+
+# Extract audio embeddings
+audio_embeddings = clap_model.get_audio_embeddings(file_paths: List[str])
+
+# Compute similarity between audio and text embeddings 
+similarities = clap_model.compute_similarity(audio_embeddings, text_embeddings)
+```
+
+- Audio Captioning
+```python
+from msclap import CLAP
+
+# Load model (Choose version 'clapcap')
+clap_model = CLAP("<PATH TO WEIGHTS>", version = 'clapcap', use_cuda=False)
+
+# Generate audio captions
+captions = clap_model.generate_caption(file_paths: List[str])
+```
+
+
+## Citation
+
+Kindly cite our work if you find it useful.
+
+[CLAP: Learning Audio Concepts from Natural Language Supervision](https://ieeexplore.ieee.org/abstract/document/10095889)
+```
+@inproceedings{CLAP2022,
+  title={Clap learning audio concepts from natural language supervision},
+  author={Elizalde, Benjamin and Deshmukh, Soham and Al Ismail, Mahmoud and Wang, Huaming},
+  booktitle={ICASSP 2023-2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+  pages={1--5},
+  year={2023},
+  organization={IEEE}
+}
+```
+
+[Natural Language Supervision for General-Purpose Audio Representations](https://arxiv.org/abs/2309.05767)
+```
+@misc{CLAP2023,
+      title={Natural Language Supervision for General-Purpose Audio Representations}, 
+      author={Benjamin Elizalde and Soham Deshmukh and Huaming Wang},
+      year={2023},
+      eprint={2309.05767},
+      archivePrefix={arXiv},
+      primaryClass={cs.SD},
+      url={https://arxiv.org/abs/2309.05767}
+}
+```
+
+## Trademarks
+
+This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 
+trademarks or logos is subject to and must follow 
+[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
+Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
+Any use of third-party trademarks or logos are subject to those third-party's policies.

pretrained_model/gpt2/README.md

@@ -0,0 +1,167 @@
+---
+language: en
+tags:
+- exbert
+
+license: mit
+---
+
+
+# GPT-2
+
+Test the whole generation capabilities here: https://transformer.huggingface.co/doc/gpt2-large
+
+Pretrained model on English language using a causal language modeling (CLM) objective. It was introduced in
+[this paper](https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf)
+and first released at [this page](https://openai.com/blog/better-language-models/).
+
+Disclaimer: The team releasing GPT-2 also wrote a
+[model card](https://github.com/openai/gpt-2/blob/master/model_card.md) for their model. Content from this model card
+has been written by the Hugging Face team to complete the information they provided and give specific examples of bias.
+
+## Model description
+
+GPT-2 is a transformers model pretrained on a very large corpus of English data in a self-supervised fashion. This
+means it was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots
+of publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely,
+it was trained to guess the next word in sentences.
+
+More precisely, inputs are sequences of continuous text of a certain length and the targets are the same sequence,
+shifted one token (word or piece of word) to the right. The model uses internally a mask-mechanism to make sure the
+predictions for the token `i` only uses the inputs from `1` to `i` but not the future tokens.
+
+This way, the model learns an inner representation of the English language that can then be used to extract features
+useful for downstream tasks. The model is best at what it was pretrained for however, which is generating texts from a
+prompt.
+
+This is the **smallest** version of GPT-2, with 124M parameters. 
+
+**Related Models:** [GPT-Large](https://huggingface.co/gpt2-large), [GPT-Medium](https://huggingface.co/gpt2-medium) and [GPT-XL](https://huggingface.co/gpt2-xl)
+
+## Intended uses & limitations
+
+You can use the raw model for text generation or fine-tune it to a downstream task. See the
+[model hub](https://huggingface.co/models?filter=gpt2) to look for fine-tuned versions on a task that interests you.
+
+### How to use
+
+You can use this model directly with a pipeline for text generation. Since the generation relies on some randomness, we
+set a seed for reproducibility:
+
+```python
+>>> from transformers import pipeline, set_seed
+>>> generator = pipeline('text-generation', model='gpt2')
+>>> set_seed(42)
+>>> generator("Hello, I'm a language model,", max_length=30, num_return_sequences=5)
+
+[{'generated_text': "Hello, I'm a language model, a language for thinking, a language for expressing thoughts."},
+ {'generated_text': "Hello, I'm a language model, a compiler, a compiler library, I just want to know how I build this kind of stuff. I don"},
+ {'generated_text': "Hello, I'm a language model, and also have more than a few of your own, but I understand that they're going to need some help"},
+ {'generated_text': "Hello, I'm a language model, a system model. I want to know my language so that it might be more interesting, more user-friendly"},
+ {'generated_text': 'Hello, I\'m a language model, not a language model"\n\nThe concept of "no-tricks" comes in handy later with new'}]
+```
+
+Here is how to use this model to get the features of a given text in PyTorch:
+
+```python
+from transformers import GPT2Tokenizer, GPT2Model
+tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
+model = GPT2Model.from_pretrained('gpt2')
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='pt')
+output = model(**encoded_input)
+```
+
+and in TensorFlow:
+
+```python
+from transformers import GPT2Tokenizer, TFGPT2Model
+tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
+model = TFGPT2Model.from_pretrained('gpt2')
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='tf')
+output = model(encoded_input)
+```
+
+### Limitations and bias
+
+The training data used for this model has not been released as a dataset one can browse. We know it contains a lot of
+unfiltered content from the internet, which is far from neutral. As the openAI team themselves point out in their
+[model card](https://github.com/openai/gpt-2/blob/master/model_card.md#out-of-scope-use-cases):
+
+> Because large-scale language models like GPT-2 do not distinguish fact from fiction, we don’t support use-cases
+> that require the generated text to be true.
+>
+> Additionally, language models like GPT-2 reflect the biases inherent to the systems they were trained on, so we do
+> not recommend that they be deployed into systems that interact with humans > unless the deployers first carry out a
+> study of biases relevant to the intended use-case. We found no statistically significant difference in gender, race,
+> and religious bias probes between 774M and 1.5B, implying all versions of GPT-2 should be approached with similar
+> levels of caution around use cases that are sensitive to biases around human attributes.
+
+Here's an example of how the model can have biased predictions:
+
+```python
+>>> from transformers import pipeline, set_seed
+>>> generator = pipeline('text-generation', model='gpt2')
+>>> set_seed(42)
+>>> generator("The White man worked as a", max_length=10, num_return_sequences=5)
+
+[{'generated_text': 'The White man worked as a mannequin for'},
+ {'generated_text': 'The White man worked as a maniser of the'},
+ {'generated_text': 'The White man worked as a bus conductor by day'},
+ {'generated_text': 'The White man worked as a plumber at the'},
+ {'generated_text': 'The White man worked as a journalist. He had'}]
+
+>>> set_seed(42)
+>>> generator("The Black man worked as a", max_length=10, num_return_sequences=5)
+
+[{'generated_text': 'The Black man worked as a man at a restaurant'},
+ {'generated_text': 'The Black man worked as a car salesman in a'},
+ {'generated_text': 'The Black man worked as a police sergeant at the'},
+ {'generated_text': 'The Black man worked as a man-eating monster'},
+ {'generated_text': 'The Black man worked as a slave, and was'}]
+```
+
+This bias will also affect all fine-tuned versions of this model.
+
+## Training data
+
+The OpenAI team wanted to train this model on a corpus as large as possible. To build it, they scraped all the web
+pages from outbound links on Reddit which received at least 3 karma. Note that all Wikipedia pages were removed from
+this dataset, so the model was not trained on any part of Wikipedia. The resulting dataset (called WebText) weights
+40GB of texts but has not been publicly released. You can find a list of the top 1,000 domains present in WebText
+[here](https://github.com/openai/gpt-2/blob/master/domains.txt).
+
+## Training procedure
+
+### Preprocessing
+
+The texts are tokenized using a byte-level version of Byte Pair Encoding (BPE) (for unicode characters) and a
+vocabulary size of 50,257. The inputs are sequences of 1024 consecutive tokens.
+
+The larger model was trained on 256 cloud TPU v3 cores. The training duration was not disclosed, nor were the exact
+details of training.
+
+## Evaluation results
+
+The model achieves the following results without any fine-tuning (zero-shot):
+
+| Dataset  | LAMBADA | LAMBADA | CBT-CN | CBT-NE | WikiText2 | PTB    | enwiki8 | text8  | WikiText103 | 1BW   |
+|:--------:|:-------:|:-------:|:------:|:------:|:---------:|:------:|:-------:|:------:|:-----------:|:-----:|
+| (metric) | (PPL)   | (ACC)   | (ACC)  | (ACC)  | (PPL)     | (PPL)  | (BPB)   | (BPC)  | (PPL)       | (PPL) |
+|          | 35.13   | 45.99   | 87.65  | 83.4   | 29.41     | 65.85  | 1.16    | 1,17   | 37.50       | 75.20 |
+
+
+### BibTeX entry and citation info
+
+```bibtex
+@article{radford2019language,
+  title={Language Models are Unsupervised Multitask Learners},
+  author={Radford, Alec and Wu, Jeff and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya},
+  year={2019}
+}
+```
+
+<a href="https://huggingface.co/exbert/?model=gpt2">
+	<img width="300px" src="https://cdn-media.huggingface.co/exbert/button.png">
+</a>

unet/pretrain_ft/config.json

@@ -0,0 +1,65 @@
+{
+  "_class_name": "UNet2DConditionModel",
+  "_diffusers_version": "0.20.0",
+  "act_fn": "silu",
+  "addition_embed_type": null,
+  "addition_embed_type_num_heads": 64,
+  "addition_time_embed_dim": null,
+  "attention_head_dim": 8,
+  "attention_type": "default",
+  "block_out_channels": [
+    128,
+    256,
+    384,
+    640
+  ],
+  "center_input_sample": false,
+  "class_embed_type": null,
+  "class_embeddings_concat": false,
+  "conv_in_kernel": 3,
+  "conv_out_kernel": 3,
+  "cross_attention_dim": 1024,
+  "cross_attention_norm": null,
+  "down_block_types": [
+    "DownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dual_cross_attention": false,
+  "encoder_hid_dim": null,
+  "encoder_hid_dim_type": null,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 8,
+  "layers_per_block": 2,
+  "mid_block_only_cross_attention": null,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_attention_heads": null,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "out_channels": 8,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_out_scale_factor": 1.0,
+  "resnet_skip_time_act": false,
+  "resnet_time_scale_shift": "default",
+  "sample_size": 128,
+  "time_cond_proj_dim": null,
+  "time_embedding_act_fn": null,
+  "time_embedding_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "transformer_layers_per_block": 1,
+  "up_block_types": [
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D",
+    "UpBlock2D"
+  ],
+  "upcast_attention": false,
+  "use_linear_projection": false
+}

unet/pretrain_ft/diffusion_pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6aad8cdb14c7ff74a72ab774e3ce23f19f6337fa22fabda4068aff7f99a35ac
+size 751830529