Bugfixes (#1045)

* Removed special character from __author__ attribute.

* Changed -v CLI arg to have a single setting, rather than multiple.

* Add retry functionality for IncompleteRead errors.

* Extract contentLength from info where possible.

* Mock open in final streams test to prevent file from being written.

* Exception handling for accessing titles of private videos.

pytube/__init__.py

@@ -4,7 +4,7 @@
 Pytube: a very serious Python library for downloading YouTube Videos.
 """
 __title__ = "pytube"
-__author__ = "Ronnie Ghose, Taylor Fox Dahlin, Nick Ficano"
+__author__ = "Ronnie Ghose, Taylor Fox Dahlin, Nick Ficano"
 __license__ = "The Unlicense (Unlicense)"
 __js__ = None
 __js_url__ = None

pytube/__main__.py

@@ -351,7 +351,20 @@ class YouTube:
         """
         if self._title:
             return self._title
-        self._title = self.player_response['videoDetails']['title']
+
+        try:
+            self._title = self.player_response['videoDetails']['title']
+        except KeyError:
+            # Check_availability will raise the correct exception in most cases
+            #  if it doesn't, ask for a report.
+            self.check_availability()
+            raise exceptions.PytubeError(
+                (
+                    f'Exception while accessing title of {self.watch_url}. '
+                    'Please file a bug report at https://github.com/pytube/pytube'
+                )
+            )
+
         return self._title
 
     @title.setter

pytube/cli.py

@@ -17,17 +17,20 @@ from pytube import CaptionQuery, Playlist, Stream, YouTube
 from pytube.helpers import safe_filename, setup_logger
 
 
+logger = logging.getLogger(__name__)
+
+
 def main():
     """Command line application to download youtube videos."""
     # noinspection PyTypeChecker
     parser = argparse.ArgumentParser(description=main.__doc__)
     args = _parse_args(parser)
-    if args.verbosity:
+    if args.verbose:
         log_filename = None
-        log_level = min(args.verbosity, 4) * 10
         if args.logfile:
             log_filename = args.logfile
-        setup_logger(logging.FATAL - log_level, log_filename=log_filename)
+        setup_logger(logging.DEBUG, log_filename=log_filename)
+        logger.debug(f'Pytube version: {__version__}')
 
     if not args.url or "youtu" not in args.url:
         parser.print_help()
@@ -113,10 +116,9 @@ def _parse_args(
     parser.add_argument(
         "-v",
         "--verbose",
-        action="count",
-        default=0,
-        dest="verbosity",
-        help="Verbosity level, use up to 4 to increase logging -vvvv",
+        action="store_true",
+        dest="verbose",
+        help="Set logger output to verbose output.",
     )
     parser.add_argument(
         "--logfile",

pytube/extract.py

@@ -532,6 +532,7 @@ def apply_descrambler(stream_data: Dict, key: str) -> None:
                     "fps": format_item["fps"] if 'video' in format_item["mimeType"] else None,
                     "bitrate": format_item.get("bitrate"),
                     "is_otf": (format_item.get("type") == otf_type),
+                    'content_length': int(format_item.get('contentLength', 0)),
                 }
                 for format_item in formats
             ]
@@ -554,6 +555,7 @@ def apply_descrambler(stream_data: Dict, key: str) -> None:
                     "fps": format_item["fps"] if 'video' in format_item["mimeType"] else None,
                     "bitrate": format_item.get("bitrate"),
                     "is_otf": (format_item.get("type") == otf_type),
+                    'content_length': int(format_item.get('contentLength', 0)),
                 }
                 for i, format_item in enumerate(formats)
             ]

pytube/request.py

@@ -1,4 +1,5 @@
 """Implements a simple wrapper around urlopen."""
+import http.client
 import json
 import logging
 import re
@@ -166,6 +167,9 @@ def stream(
                     pass
                 else:
                     raise
+            except http.client.IncompleteRead:
+                # Allow retries on IncompleteRead errors for unreliable connections
+                pass
             else:
                 # On a successful request, break from loop
                 break

pytube/streams.py

@@ -62,7 +62,7 @@ class Stream:
         self.is_otf: bool = stream["is_otf"]
         self.bitrate: Optional[int] = stream["bitrate"]
 
-        self._filesize: Optional[int] = None  # filesize in bytes
+        self._filesize: Optional[int] = stream['content_length']  # filesize in bytes
 
         # Additional information about the stream format, such as resolution,
         # frame rate, and whether the stream is live (HLS) or 3D.
@@ -147,7 +147,7 @@ class Stream:
         :returns:
             Filesize (in bytes) of the stream.
         """
-        if self._filesize is None:
+        if self._filesize == 0:
             try:
                 self._filesize = request.filesize(self.url)
             except HTTPError as e:

tests/test_cli.py

@@ -1,4 +1,5 @@
 import argparse
+import logging
 import os
 from unittest import mock
 from unittest.mock import MagicMock, patch
@@ -139,7 +140,7 @@ def test_parse_args_falsey():
     assert args.build_playback_report is False
     assert args.itag is None
     assert args.list is False
-    assert args.verbosity == 0
+    assert args.verbose is False
 
 
 def test_parse_args_truthy():
@@ -160,7 +161,7 @@ def test_parse_args_truthy():
     assert args.build_playback_report is True
     assert args.itag == 10
     assert args.list is True
-    assert args.verbosity == 3
+    assert args.verbose is True
 
 
 @mock.patch("pytube.cli.setup_logger", return_value=None)
@@ -173,7 +174,7 @@ def test_main_logging_setup(setup_logger):
     with pytest.raises(SystemExit):
         cli.main()
     # Then
-    setup_logger.assert_called_with(40, log_filename=None)
+    setup_logger.assert_called_with(logging.DEBUG, log_filename=None)
 
 
 @mock.patch("pytube.cli.YouTube", return_value=None)

tests/test_streams.py

@@ -27,22 +27,16 @@ def test_stream_to_buffer(mock_request, cipher_signature):
     assert buffer.write.call_count == 3
 
 
-@mock.patch(
-    "pytube.streams.request.head", MagicMock(return_value={"content-length": "6796391"})
-)
 def test_filesize(cipher_signature):
-    assert cipher_signature.streams[0].filesize == 6796391
+    assert cipher_signature.streams[0].filesize == 28282013
 
 
-@mock.patch(
-    "pytube.streams.request.head", MagicMock(return_value={"content-length": "6796391"})
-)
 def test_filesize_approx(cipher_signature):
     stream = cipher_signature.streams[0]
 
     assert stream.filesize_approx == 28309811
     stream.bitrate = None
-    assert stream.filesize_approx == 6796391
+    assert stream.filesize_approx == 28282013
 
 
 def test_default_filename(cipher_signature):
@@ -345,14 +339,11 @@ def test_segmented_stream_on_404(cipher_signature):
             ]
 
             # Request order for stream:
-            # Filesize:
-            #   1. head(url) -> 404
-            #   2. get(url&sn=0)
-            #   3. head(url&sn=[1,2,3])
-            # Download:
-            #   4. info(url) -> 404
-            #   5. get(url&sn=0)
-            #   6. get(url&sn=[1,2,3])
+            #   1. get(url&sn=0)
+            #   2. head(url&sn=[1,2,3])
+            #   3. info(url) -> 404
+            #   4. get(url&sn=0)
+            #   5. get(url&sn=[1,2,3])
 
             # Handle filesize requests
             mock_head.side_effect = [
@@ -363,7 +354,6 @@ def test_segmented_stream_on_404(cipher_signature):
             # Each response must be followed by None, to break iteration
             #  in the stream() function
             mock_url_open_object.read.side_effect = [
-                responses[0], None,
                 responses[0], None,
                 responses[1], None,
                 responses[2], None,
@@ -394,5 +384,6 @@ def test_segmented_only_catches_404(cipher_signature):
     stream = cipher_signature.streams.filter(adaptive=True)[0]
     with mock.patch('pytube.request.head') as mock_head:
         mock_head.side_effect = HTTPError('', 403, 'Forbidden', '', '')
-        with pytest.raises(HTTPError):
-            stream.download()
+        with mock.patch("pytube.streams.open", mock.mock_open(), create=True):
+            with pytest.raises(HTTPError):
+                stream.download()