Super-squash branch 'main' using huggingface_hub

Co-authored-by: joaogante <joaogante@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,169 @@
+# Initially taken from Github's Python gitignore file
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# tests and logs
+tests/fixtures/cached_*_text.txt
+logs/
+lightning_logs/
+lang_code_data/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# vscode
+.vs
+.vscode
+
+# Pycharm
+.idea
+
+# TF code
+tensorflow_code
+
+# Models
+proc_data
+
+# examples
+runs
+/runs_old
+/wandb
+/examples/runs
+/examples/**/*.args
+/examples/rag/sweep
+
+# data
+/data
+serialization_dir
+
+# emacs
+*.*~
+debug.env
+
+# vim
+.*.swp
+
+#ctags
+tags
+
+# pre-commit
+.pre-commit*
+
+# .lock
+*.lock
+
+# DS_Store (MacOS)
+.DS_Store
+
+# ruff
+.ruff_cache

README.md

@@ -0,0 +1,16 @@
+---
+title: text2text Instruct playground
+emoji: 🗨️⌛
+colorFrom: purple
+colorTo: pink
+sdk: gradio
+sdk_version: 4.44.1
+app_file: app.py
+pinned: false
+duplicated_from: joaogante/transformers_streaming
+license: apache-2.0
+---
+
+waddup
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,168 @@
+from threading import Thread
+import logging
+import time
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s - %(message)s",
+)
+
+import torch
+import gradio as gr
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, TextIteratorStreamer
+
+model_id = "pszemraj/tFINE-850m-24x24-v0.5-instruct-L1"
+torch_device = "cuda" if torch.cuda.is_available() else "cpu"
+logging.info(f"Running on device:\t {torch_device}")
+logging.info(f"CPU threads:\t {torch.get_num_threads()}")
+
+
+if torch_device == "cuda":
+    model = AutoModelForSeq2SeqLM.from_pretrained(
+        model_id, load_in_8bit=True, device_map="auto"
+    )
+else:
+    model = AutoModelForSeq2SeqLM.from_pretrained(model_id, torch_dtype=torch.bfloat16)
+try:
+    model = torch.compile(model)
+except Exception as e:
+    logging.error(f"Unable to compile model:\t{e}")
+    
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+
+def run_generation(
+    user_text,
+    top_p,
+    temperature,
+    top_k,
+    max_new_tokens,
+    repetition_penalty=1.1,
+    length_penalty=1.0,
+    no_repeat_ngram_size=4,
+    use_generation_config=False,
+):
+    st = time.perf_counter()
+    # Get the model and tokenizer, and tokenize the user text.
+    model_inputs = tokenizer([user_text], return_tensors="pt").to(torch_device)
+
+    # Start generation on a separate thread, so that we don't block the UI. The text is pulled from the streamer
+    # in the main thread. Adds timeout to the streamer to handle exceptions in the generation thread.
+    streamer = TextIteratorStreamer(
+        tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True
+    )
+    generate_kwargs = dict(
+        model_inputs,
+        streamer=streamer,
+        max_new_tokens=max_new_tokens,
+        do_sample=True,
+        num_beams=1,
+        top_p=top_p,
+        temperature=float(temperature),
+        top_k=top_k,
+        repetition_penalty=repetition_penalty,
+        length_penalty=length_penalty,
+        no_repeat_ngram_size=no_repeat_ngram_size,
+        renormalize_logits=True,
+    )
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
+
+    # Pull the generated text from the streamer, and update the model output.
+    model_output = ""
+    for new_text in streamer:
+        model_output += new_text
+        yield model_output
+    logging.info("Total rt:\t{rt} sec".format(rt=round(time.perf_counter() - st, 3)))
+    return model_output
+
+
+def reset_textbox():
+    return gr.update(value="")
+
+
+with gr.Blocks() as demo:
+    duplicate_link = (
+        "https://huggingface.co/spaces/joaogante/transformers_streaming?duplicate=true"
+    )
+    gr.Markdown(
+        "# 🤗 Transformers 🔥Streaming🔥 on Gradio\n"
+        "This demo showcases the use of the "
+        "[streaming feature](https://huggingface.co/docs/transformers/main/en/generation_strategies#streaming) "
+        "of 🤗 Transformers with Gradio to generate text in real-time. It uses "
+        f"[{model_id}](https://huggingface.co/{model_id}) and the Spaces free compute tier.\n\n"
+        f"Feel free to [duplicate this Space]({duplicate_link}) to try your own models or use this space as a "
+        "template! 💛"
+    )
+    gr.Markdown("---")
+    with gr.Row():
+        with gr.Column(scale=4):
+            user_text = gr.Textbox(
+                value="How to become a polar bear tamer?",
+                label="User input",
+            )
+            model_output = gr.Textbox(label="Model output", lines=10, interactive=False)
+            button_submit = gr.Button(value="Submit", variant="primary")
+
+        with gr.Column(scale=1):
+            max_new_tokens = gr.Slider(
+                minimum=32,
+                maximum=1024,
+                value=256,
+                step=32,
+                interactive=True,
+                label="Max New Tokens",
+            )
+            top_p = gr.Slider(
+                minimum=0.05,
+                maximum=1.0,
+                value=0.95,
+                step=0.05,
+                interactive=True,
+                label="Top-p (nucleus sampling)",
+            )
+            top_k = gr.Slider(
+                minimum=1,
+                maximum=50,
+                value=50,
+                step=1,
+                interactive=True,
+                label="Top-k",
+            )
+            temperature = gr.Slider(
+                minimum=0.1,
+                maximum=1.4,
+                value=0.3,
+                step=0.05,
+                interactive=True,
+                label="Temperature",
+            )
+            repetition_penalty = gr.Slider(
+                minimum=0.9,
+                maximum=2.5,
+                value=1.1,
+                step=0.1,
+                interactive=True,
+                label="Repetition Penalty",
+            )
+            length_penalty = gr.Slider(
+                minimum=0.8,
+                maximum=1.5,
+                value=1.0,
+                step=0.1,
+                interactive=True,
+                label="Length Penalty",
+            )
+
+    user_text.submit(
+        run_generation,
+        [user_text, top_p, temperature, top_k, max_new_tokens, repetition_penalty, length_penalty],
+        model_output,
+    )
+    button_submit.click(
+        run_generation,
+        [user_text, top_p, temperature, top_k, max_new_tokens, repetition_penalty, length_penalty],
+        model_output,
+    )
+
+    demo.queue(max_size=10).launch()

requirements.txt

@@ -0,0 +1,6 @@
+accelerate
+bitsandbytes
+torch
+transformers
+gradio
+sentencepiece