[Feature] Video metadata (#809)

* Added accept-language to request headers to coerce certain strings sent by YouTube into english (e.g. 'This video is private.'
* Implemented metadata class.

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
 
     strategy:
       matrix:
-        python: [3.5, 3.6, 3.7, 3.8]
+        python: [3.6, 3.7, 3.8, 3.9]
 
     steps:
       - name: Checkout repo

pytube/__main__.py

@@ -27,6 +27,7 @@ from pytube.extract import apply_descrambler
 from pytube.extract import apply_signature
 from pytube.extract import get_ytplayer_config
 from pytube.helpers import install_proxy
+from pytube.metadata import YouTubeMetadata
 from pytube.monostate import Monostate
 from pytube.monostate import OnComplete
 from pytube.monostate import OnProgress
@@ -60,23 +61,17 @@ class YouTube:
 
         """
         self.js: Optional[str] = None  # js fetched by js_url
-        self.js_url: Optional[
-            str
-        ] = None  # the url to the js, parsed from watch html
+        self.js_url: Optional[str] = None  # the url to the js, parsed from watch html
 
         # note: vid_info may eventually be removed. It sounds like it once had
         # additional formats, but that doesn't appear to still be the case.
 
         # the url to vid info, parsed from watch html
         self.vid_info_url: Optional[str] = None
-        self.vid_info_raw: Optional[
-            str
-        ] = None  # content fetched by vid_info_url
+        self.vid_info_raw: Optional[str] = None  # content fetched by vid_info_url
         self.vid_info: Optional[Dict] = None  # parsed content of vid_info_raw
 
-        self.watch_html: Optional[
-            str
-        ] = None  # the html of /watch?v=<video_id>
+        self.watch_html: Optional[str] = None  # the html of /watch?v=<video_id>
         self.embed_html: Optional[str] = None
         self.player_config_args: Dict = {}  # inline js in the html containing
         self.player_response: Dict = {}
@@ -85,6 +80,10 @@ class YouTube:
 
         self.fmt_streams: List[Stream] = []
 
+        self.initial_data_raw = None
+        self.initial_data = {}
+        self._metadata: Optional[YouTubeMetadata] = None
+
         # video_id part of /watch?v=<video_id>
         self.video_id = extract.video_id(url)
 
@@ -187,6 +186,9 @@ class YouTube:
                 video_id=self.video_id, watch_url=self.watch_url
             )
 
+        self.initial_data_raw = extract.initial_data(self.watch_html)
+        self.initial_data = json.loads(self.initial_data_raw)
+
         self.vid_info_raw = request.get(self.vid_info_url)
         if not self.age_restricted:
             self.js_url = extract.js_url(self.watch_html)
@@ -287,9 +289,7 @@ class YouTube:
         :rtype: str
 
         """
-        return self.player_response.get("videoDetails", {}).get(
-            "shortDescription"
-        )
+        return self.player_response.get("videoDetails", {}).get("shortDescription")
 
     @property
     def rating(self) -> float:
@@ -298,9 +298,7 @@ class YouTube:
         :rtype: float
 
         """
-        return self.player_response.get("videoDetails", {}).get(
-            "averageRating"
-        )
+        return self.player_response.get("videoDetails", {}).get("averageRating")
 
     @property
     def length(self) -> int:
@@ -338,6 +336,18 @@ class YouTube:
             "author", "unknown"
         )
 
+    @property
+    def metadata(self) -> Optional[YouTubeMetadata]:
+        """Get the metadata for the video.
+
+        :rtype: YouTubeMetadata
+        """
+        if self._metadata:
+            return self._metadata
+        else:
+            self._metadata = extract.metadata(self.initial_data)
+            return self._metadata
+
     def register_on_progress_callback(self, func: OnProgress):
         """Register a download progress callback function post initialization.
 

pytube/extract.py

@@ -8,6 +8,7 @@ from datetime import datetime
 from typing import Any
 from typing import Dict
 from typing import List
+from typing import Optional
 from typing import Tuple
 from urllib.parse import parse_qs
 from urllib.parse import parse_qsl
@@ -19,6 +20,7 @@ from pytube.cipher import Cipher
 from pytube.exceptions import LiveStreamError
 from pytube.exceptions import RegexMatchError
 from pytube.helpers import regex_search
+from pytube.metadata import YouTubeMetadata
 
 logger = logging.getLogger(__name__)
 
@@ -396,3 +398,58 @@ def apply_descrambler(stream_data: Dict, key: str) -> None:
         ]
 
     logger.debug("applying descrambler")
+
+
+def initial_data(watch_html: str) -> str:
+    """Extract the ytInitialData json from the watch_html page.
+
+    This mostly contains metadata necessary for rendering the page on-load,
+    such as video information, copyright notices, etc.
+
+    @param watch_html: Html of the watch page
+    @return:
+    """
+    initial_data_pattern = r"window\[['\"]ytInitialData['\"]]\s*=\s*([^\n]+)"
+    try:
+        match = regex_search(initial_data_pattern, watch_html, 1)
+    except RegexMatchError:
+        return "{}"
+    else:
+        return match[:-1]
+
+
+def metadata(initial_data) -> Optional[YouTubeMetadata]:
+    """Get the informational metadata for the video.
+
+    e.g.:
+    [
+        {
+            'Song': '강남스타일(Gangnam Style)',
+            'Artist': 'PSY',
+            'Album': 'PSY SIX RULES Pt.1',
+            'Licensed to YouTube by': 'YG Entertainment Inc. [...]'
+        }
+    ]
+
+    :rtype: YouTubeMetadata
+    """
+    try:
+        metadata_rows: List = initial_data["contents"]["twoColumnWatchNextResults"][
+            "results"]["results"]["contents"][1]["videoSecondaryInfoRenderer"][
+            "metadataRowContainer"]["metadataRowContainerRenderer"]["rows"]
+    except (KeyError, IndexError):
+        # If there's an exception accessing this data, it probably doesn't exist.
+        return YouTubeMetadata([])
+
+    # Rows appear to only have "metadataRowRenderer" or "metadataRowHeaderRenderer"
+    #  and we only care about the former, so we filter the others
+    metadata_rows = filter(
+        lambda x: "metadataRowRenderer" in x.keys(),
+        metadata_rows
+    )
+
+    # We then access the metadataRowRenderer key in each element
+    #  and build a metadata object from this new list
+    metadata_rows = [x["metadataRowRenderer"] for x in metadata_rows]
+
+    return YouTubeMetadata(metadata_rows)

pytube/metadata.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""This module contains the YouTubeMetadata class."""
+import json
+from typing import Dict
+from typing import List
+from typing import Optional
+
+
+class YouTubeMetadata:
+    def __init__(self, metadata: List):
+        self._raw_metadata: List = metadata
+        self._metadata = [{}]
+
+        for el in metadata:
+            # We only add metadata to the dict if it has a simpleText title.
+            if 'title' in el and 'simpleText' in el['title']:
+                metadata_title = el['title']['simpleText']
+            else:
+                continue
+
+            contents = el['contents'][0]
+            if 'simpleText' in contents:
+                self._metadata[-1][metadata_title] = contents['simpleText']
+            elif 'runs' in contents:
+                self._metadata[-1][metadata_title] = contents['runs'][0]['text']
+
+            # Upon reaching a dividing line, create a new grouping
+            if el.get('hasDividerLine', False):
+                self._metadata.append({})
+
+        # If we happen to create an empty dict at the end, drop it
+        if self._metadata[-1] == {}:
+            self._metadata = self._metadata[:-1]
+
+    def __iter__(self):
+        for el in self._metadata:
+            yield el
+
+    def __str__(self):
+        return json.dumps(self._metadata)
+
+    @property
+    def raw_metadata(self) -> Optional[Dict]:
+        return self._raw_metadata
+
+    @property
+    def metadata(self):
+        return self._metadata

pytube/request.py

@@ -16,7 +16,7 @@ default_range_size = 9437184  # 9MB
 
 
 def _execute_request(url, method=None, headers=None):
-    base_headers = {"User-Agent": "Mozilla/5.0"}
+    base_headers = {"User-Agent": "Mozilla/5.0", "accept-language": "en-US,en"}
     if headers:
         base_headers.update(headers)
     if url.lower().startswith("http"):

tests/test_extract.py

@@ -102,3 +102,13 @@ def test_signature_cipher_does_not_error(stream_dict):
     config_args = extract.get_ytplayer_config(stream_dict)['args']
     extract.apply_descrambler(config_args, "url_encoded_fmt_stream_map")
     assert "s" in config_args["url_encoded_fmt_stream_map"][0].keys()
+
+
+def test_initial_data_missing():
+    initial_data = extract.initial_data('')
+    assert initial_data == "{}"
+
+
+def test_initial_data(stream_dict):
+    initial_data = extract.initial_data(stream_dict)
+    assert 'contents' in initial_data

tests/test_metadata.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+"""Unit tests for the :module:`metadata <metadata>` module."""
+import json
+from pytube import extract
+
+
+def test_extract_metadata_empty():
+    ytmd = extract.metadata({})
+    assert ytmd._raw_metadata == []
+
+
+def test_metadata_from_initial_data(stream_dict):
+    initial_data = extract.initial_data(stream_dict)
+    ytmd = extract.metadata(json.loads(initial_data))
+    assert len(ytmd.raw_metadata) > 0
+    assert 'contents' in ytmd.raw_metadata[0]
+    assert len(ytmd.metadata) > 0
+    assert 'Song' in ytmd.metadata[0]