Fix for #763 (#767)

* Fix KeyError: 'assets', and minor refactor
* Fixed tests. Changes to YT broke a single test, and updating the mock had cascading effects on other tests. This updates the tests to reflect a more recent YT page, and introduces os-agnostic file paths for testing on windows.
* Fixed flake8 issues.

pytube/__main__.py

@@ -293,7 +293,7 @@ class YouTube:
         """
         return self.player_response.get("videoDetails", {}).get(
             "shortDescription"
-        ) or extract._get_vid_descr(self.watch_html)
+        )
 
     @property
     def rating(self) -> float:

pytube/extract.py

@@ -4,11 +4,9 @@ import json
 import logging
 import re
 from collections import OrderedDict
-from html.parser import HTMLParser
 from typing import Any
 from typing import Dict
 from typing import List
-from typing import Optional
 from typing import Tuple
 from urllib.parse import parse_qs
 from urllib.parse import parse_qsl
@@ -17,7 +15,6 @@ from urllib.parse import unquote
 from urllib.parse import urlencode
 
 from pytube.cipher import Cipher
-from pytube.exceptions import HTMLParseError
 from pytube.exceptions import LiveStreamError
 from pytube.exceptions import RegexMatchError
 from pytube.helpers import regex_search
@@ -25,36 +22,6 @@ from pytube.helpers import regex_search
 logger = logging.getLogger(__name__)
 
 
-class PytubeHTMLParser(HTMLParser):
-    in_vid_descr = False
-    in_vid_descr_br = False
-    vid_descr = ""
-
-    def handle_starttag(self, tag, attrs):
-        if tag == "p":
-            for attr in attrs:
-                if attr[0] == "id" and attr[1] == "eow-description":
-                    self.in_vid_descr = True
-
-    def handle_endtag(self, tag):
-        if self.in_vid_descr and tag == "p":
-            self.in_vid_descr = False
-
-    def handle_startendtag(self, tag, attrs):
-        if self.in_vid_descr and tag == "br":
-            self.in_vid_descr_br = True
-
-    def handle_data(self, data):
-        if self.in_vid_descr_br:
-            self.vid_descr += f"\n{data}"
-            self.in_vid_descr_br = False
-        elif self.in_vid_descr:
-            self.vid_descr += data
-
-    def error(self, message):
-        raise HTMLParseError(message)
-
-
 def is_age_restricted(watch_html: str) -> bool:
     """Check if content is age restricted.
 
@@ -151,7 +118,7 @@ def js_url(html: str) -> str:
     :param str html:
         The html contents of the watch page.
     """
-    base_js = get_ytplayer_config(html)["assets"]["js"]
+    base_js = get_ytplayer_js(html)
     return "https://youtube.com" + base_js
 
 
@@ -182,6 +149,31 @@ def mime_type_codec(mime_type_codec: str) -> Tuple[str, List[str]]:
     return mime_type, [c.strip() for c in codecs.split(",")]
 
 
+def get_ytplayer_js(html: str) -> Any:
+    """Get the YouTube player base JavaScript path.
+
+    :param str html
+        The html contents of the watch page.
+    :rtype: str
+    :returns:
+        Path to YouTube's base.js file.
+    """
+    js_url_patterns = [
+        r"\"jsUrl\":\"([^\"]*)\"",
+    ]
+    for pattern in js_url_patterns:
+        regex = re.compile(pattern)
+        function_match = regex.search(html)
+        if function_match:
+            logger.debug("finished regex search, matched: %s", pattern)
+            yt_player_js = function_match.group(1)
+            return yt_player_js
+
+    raise RegexMatchError(
+        caller="get_ytplayer_js", pattern="js_url_patterns"
+    )
+
+
 def get_ytplayer_config(html: str) -> Any:
     """Get the YouTube player configuration data from the watch html.
 
@@ -215,13 +207,6 @@ def get_ytplayer_config(html: str) -> Any:
     )
 
 
-def _get_vid_descr(html: Optional[str]) -> str:
-    html_parser = PytubeHTMLParser()
-    if html:
-        html_parser.feed(html)
-    return html_parser.vid_descr
-
-
 def apply_signature(config_args: Dict, fmt: str, js: str) -> None:
     """Apply the decrypted signature to the stream manifest.
 
@@ -316,11 +301,11 @@ def apply_descrambler(stream_data: Dict, key: str) -> None:
         except KeyError:
             cipher_url = [
                 parse_qs(
-                    formats[i][
+                    data[
                         "cipher" if "cipher" in data.keys() else "signatureCipher"
                     ]
                 )
-                for i, data in enumerate(formats)
+                for data in formats
             ]
             stream_data[key] = [
                 {

tests/test_captions.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import os
 from unittest import mock
 from unittest.mock import MagicMock
 from unittest.mock import mock_open
@@ -71,7 +72,7 @@ def test_download(srt):
         )
         caption.download("title")
         assert (
-            open_mock.call_args_list[0][0][0].split("/")[-1] == "title (en).srt"
+            open_mock.call_args_list[0][0][0].split(os.path.sep)[-1] == "title (en).srt"
         )
 
 
@@ -89,7 +90,7 @@ def test_download_with_prefix(srt):
         )
         caption.download("title", filename_prefix="1 ")
         assert (
-            open_mock.call_args_list[0][0][0].split("/")[-1]
+            open_mock.call_args_list[0][0][0].split(os.path.sep)[-1]
             == "1 title (en).srt"
         )
 
@@ -108,7 +109,7 @@ def test_download_with_output_path(srt):
             }
         )
         file_path = caption.download("title", output_path="blah")
-        assert file_path == "/target/title (en).srt"
+        assert file_path == os.path.join("/target","title (en).srt")
         captions.target_directory.assert_called_with("blah")
 
 
@@ -126,7 +127,7 @@ def test_download_xml_and_trim_extension(xml):
         )
         caption.download("title.xml", srt=False)
         assert (
-            open_mock.call_args_list[0][0][0].split("/")[-1] == "title (en).xml"
+            open_mock.call_args_list[0][0][0].split(os.path.sep)[-1] == "title (en).xml"
         )
 
 

tests/test_extract.py

@@ -38,7 +38,7 @@ def test_info_url_age_restricted(cipher_signature):
 
 def test_js_url(cipher_signature):
     expected = (
-        "https://youtube.com/yts/jsbin/player_ias-vflWQEEag/en_US/base.js"
+        "https://youtube.com/s/player/4a1799bd/player_ias.vflset/en_US/base.js"
     )
     result = extract.js_url(cipher_signature.watch_html)
     assert expected == result
@@ -52,25 +52,6 @@ def test_non_age_restricted(cipher_signature):
     assert not extract.is_age_restricted(cipher_signature.watch_html)
 
 
-def test_get_vid_desc(cipher_signature):
-    expected = (
-        "PSY - ‘I LUV IT’ M/V @ https://youtu.be/Xvjnoagk6GU\n"
-        "PSY - ‘New Face’ M/V @https://youtu.be/OwJPPaEyqhI\n"
-        "PSY - 8TH ALBUM '4X2=8' on iTunes @\n"
-        "https://smarturl.it/PSY_8thAlbum\n"
-        "PSY - GANGNAM STYLE(강남스타일) on iTunes @ http://smarturl.it/PsyGangnam\n"
-        "#PSY #싸이 #GANGNAMSTYLE #강남스타일\n"
-        "More about PSY@\nhttp://www.youtube.com/officialpsy\n"
-        "http://www.facebook.com/officialpsy\n"
-        "http://twitter.com/psy_oppa\n"
-        "https://www.instagram.com/42psy42\n"
-        "http://iTunes.com/PSY\n"
-        "http://sptfy.com/PSY\n"
-        "http://weibo.com/psyoppa"
-    )
-    assert extract._get_vid_descr(cipher_signature.watch_html) == expected
-
-
 def test_mime_type_codec():
     mime_type, mime_subtype = extract.mime_type_codec(
         'audio/webm; codecs="opus"'

tests/test_helpers.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import os
 from unittest import mock
 
 import pytest
@@ -61,7 +62,7 @@ def test_cache():
 @mock.patch("os.getcwd", return_value="/cwd")
 @mock.patch("os.makedirs")
 def test_target_directory_with_relative_path(_, __, makedirs):  # noqa: PT019
-    assert target_directory("test") == "/cwd/test"
+    assert target_directory("test") == os.path.join("/cwd", "test")
     makedirs.assert_called()
 
 

tests/test_query.py

@@ -144,7 +144,7 @@ def test_get_highest_resolution(cipher_signature):
 def test_filter_is_dash(cipher_signature):
     streams = cipher_signature.streams.filter(is_dash=False)
     itags = [s.itag for s in streams]
-    assert itags == [18, 398, 397, 396, 395, 394]
+    assert itags == [18, 399, 398, 397, 396, 395, 394]
 
 
 def test_get_audio_only(cipher_signature):
@@ -156,13 +156,13 @@ def test_get_audio_only_with_subtype(cipher_signature):
 
 
 def test_sequence(cipher_signature):
-    assert len(cipher_signature.streams) == 22
+    assert len(cipher_signature.streams) == 23
     assert cipher_signature.streams[0] is not None
 
 
 def test_otf(cipher_signature):
     non_otf = cipher_signature.streams.otf()
-    assert len(non_otf) == 22
+    assert len(non_otf) == 23
 
     otf = cipher_signature.streams.otf(True)
     assert len(otf) == 0

tests/test_streams.py

@@ -63,7 +63,7 @@ def test_title(cipher_signature):
 
 
 def test_expiration(cipher_signature):
-    assert cipher_signature.streams[0].expiration == datetime(2020, 1, 16, 5, 12, 5)
+    assert cipher_signature.streams[0].expiration == datetime(2020, 10, 24, 11, 7, 41)
 
 
 def test_caption_tracks(presigned_video):
@@ -93,27 +93,9 @@ def test_description(cipher_signature):
     )
     assert cipher_signature.description == expected
 
-    cipher_signature.player_response = {}
-    expected = (
-        "PSY - ‘I LUV IT’ M/V @ https://youtu.be/Xvjnoagk6GU\n"
-        "PSY - ‘New Face’ M/V @https://youtu.be/OwJPPaEyqhI\n"
-        "PSY - 8TH ALBUM '4X2=8' on iTunes @\n"
-        "https://smarturl.it/PSY_8thAlbum\n"
-        "PSY - GANGNAM STYLE(강남스타일) on iTunes @ http://smarturl.it/PsyGangnam\n"
-        "#PSY #싸이 #GANGNAMSTYLE #강남스타일\n"
-        "More about PSY@\nhttp://www.youtube.com/officialpsy\n"
-        "http://www.facebook.com/officialpsy\n"
-        "http://twitter.com/psy_oppa\n"
-        "https://www.instagram.com/42psy42\n"
-        "http://iTunes.com/PSY\n"
-        "http://sptfy.com/PSY\n"
-        "http://weibo.com/psyoppa"
-    )
-    assert cipher_signature.description == expected
-
 
 def test_rating(cipher_signature):
-    assert cipher_signature.rating == 4.522203
+    assert cipher_signature.rating == 4.5375643
 
 
 def test_length(cipher_signature):
@@ -121,7 +103,7 @@ def test_length(cipher_signature):
 
 
 def test_views(cipher_signature):
-    assert cipher_signature.views == 3494704859
+    assert cipher_signature.views == 3830838693
 
 
 @mock.patch(
@@ -149,7 +131,10 @@ def test_download_with_prefix(cipher_signature):
     with mock.patch("pytube.streams.open", mock.mock_open(), create=True):
         stream = cipher_signature.streams[0]
         file_path = stream.download(filename_prefix="prefix")
-        assert file_path == "/target/prefixPSY - GANGNAM STYLE(강남스타일) MV.mp4"
+        assert file_path == os.path.join(
+            "/target",
+            "prefixPSY - GANGNAM STYLE(강남스타일) MV.mp4"
+        )
 
 
 @mock.patch(
@@ -164,7 +149,10 @@ def test_download_with_filename(cipher_signature):
     with mock.patch("pytube.streams.open", mock.mock_open(), create=True):
         stream = cipher_signature.streams[0]
         file_path = stream.download(filename="cool name bro")
-        assert file_path == "/target/cool name bro.mp4"
+        assert file_path == os.path.join(
+            "/target",
+            "cool name bro.mp4"
+        )
 
 
 @mock.patch(
@@ -181,7 +169,10 @@ def test_download_with_existing(cipher_signature):
         stream = cipher_signature.streams[0]
         os.path.getsize = Mock(return_value=stream.filesize)
         file_path = stream.download()
-        assert file_path == "/target/PSY - GANGNAM STYLE(강남스타일) MV.mp4"
+        assert file_path == os.path.join(
+            "/target",
+            "PSY - GANGNAM STYLE(강남스타일) MV.mp4"
+        )
         assert not request.stream.called
 
 
@@ -199,7 +190,10 @@ def test_download_with_existing_no_skip(cipher_signature):
         stream = cipher_signature.streams[0]
         os.path.getsize = Mock(return_value=stream.filesize)
         file_path = stream.download(skip_existing=False)
-        assert file_path == "/target/PSY - GANGNAM STYLE(강남스타일) MV.mp4"
+        assert file_path == os.path.join(
+            "/target",
+            "PSY - GANGNAM STYLE(강남스타일) MV.mp4"
+        )
         assert request.stream.called