Duplicate from remzicam/ted_talks_summarizer

Co-authored-by: rc <remzicam@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: Ted Talks Summarizer
+emoji: 🌖
+colorFrom: pink
+colorTo: gray
+sdk: gradio
+sdk_version: 3.15.0
+app_file: app.py
+pinned: false
+license: other
+duplicated_from: remzicam/ted_talks_summarizer
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,108 @@
+"""TED Talks Summarizer App."""
+
+from re import sub
+
+from gradio import Interface, Textbox
+from requests import get
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, pipeline
+
+repo_id = "pszemraj/led-base-book-summary"
+
+model = AutoModelForSeq2SeqLM.from_pretrained(
+    repo_id,
+    low_cpu_mem_usage=True,
+)
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id)
+
+summarizer = pipeline("summarization", model=model, tokenizer=tokenizer)
+
+
+def clean_text(text: str) -> str:
+    """Cleans subtitle text of ted talks.
+
+    Args:
+        text (str): subtitle of ted talk
+
+    Returns:
+        cleaned_text (str): cleaned version of subtitle text
+    """
+    # remove string inside parantheses (i.e appluse)
+    text = sub(r"\(.*\)", "", text)
+    # format text by splitting/removing new lines
+    text = text.split("\n")[1:]
+    # remove empty strings
+    text = list(filter(None, text))
+    # remove timestamps as they contains pattern of "-->"
+    cleaned_text = " ".join([x.strip() for x in text if "-->" not in x])
+    return cleaned_text
+
+
+def ted_talk_transcriber(link: str) -> str:
+    """Creates transcription of ted talks from url.
+
+    Args:
+        link (str): url link of ted talks
+
+    Returns:
+        raw_text (str): raw transcription of the ted talk
+    """
+    # request link of the talk
+    page = get(link)
+    # extract unique talk id to reach subtitle file
+    talk_id = str(page.content).split("project_masters/")[1].split("/")[0]
+    raw_text = get(
+        f"https://hls.ted.com/project_masters/{talk_id}/subtitles/en/full.vtt"
+    ).text
+    return raw_text
+
+
+def text_summarizer(text: str) -> str:
+    """Summarizes given text.
+
+    Args:
+        text (str): ted talks transcription
+
+    Returns:
+        str: summary
+    """
+    result = summarizer(
+        text,
+        min_length=8,
+        max_length=256,
+        no_repeat_ngram_size=3,
+        encoder_no_repeat_ngram_size=3,
+        repetition_penalty=3.5,
+        num_beams=4,
+        do_sample=False,
+        early_stopping=True,
+    )
+    return result[0]["summary_text"]
+
+
+def main(link: str) -> str:
+    """Summarizes ted talks given link.
+
+    Args:
+        link (str): url link of ted talks
+
+    Returns:
+        str: summary
+    """
+    raw_text = ted_talk_transcriber(link)
+    cleaned_transcript = clean_text(raw_text)
+    return text_summarizer(cleaned_transcript)
+
+
+logo = "<center><img src='file/TED.png' width=180px></center>"
+
+Interface(
+    main,
+    inputs=Textbox(label="Type the TED Talks link"),
+    examples=[
+    "https://www.ted.com/talks/jen_gunter_the_truth_about_yeast_in_your_body"
+             ],
+    outputs=Textbox(label="Summary"),
+    allow_flagging="never",
+    description=logo,
+).launch()

requirements.txt

@@ -0,0 +1,4 @@
+accelerate
+--find-links https://download.pytorch.org/whl/torch_stable.html
+torch==1.13.1+cpu
+transformers