yt_noapi.py 6.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177
  1. #
  2. # This file is part of stov, written by Helmut Pozimski 2012-2021.
  3. #
  4. # stov is free software: you can redistribute it and/or modify
  5. # it under the terms of the GNU General Public License as published by
  6. # the Free Software Foundation, version 2 of the License.
  7. #
  8. # stov is distributed in the hope that it will be useful,
  9. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. # GNU General Public License for more details.
  12. #
  13. # You should have received a copy of the GNU General Public License
  14. # along with stov. If not, see <http://www.gnu.org/licenses/>.
  15. # -*- coding: utf8 -*-
  16. """This module provides all classes and methods that were provided by the
  17. youtubeAPI module earlier."""
  18. import logging
  19. import subprocess
  20. import urllib.error
  21. import urllib.parse
  22. import urllib.request
  23. import lxml.html
  24. from lib_stov import configuration
  25. from lib_stov import stov_exceptions
  26. from lib_stov import youtubedl_wrapper
  27. LOGGER = logging.getLogger("stov")
  28. class YtChannel:
  29. """Stores the relevant attributes of a youtube channel."""
  30. def __init__(self, _type, title, videos=None):
  31. if videos is None:
  32. videos = []
  33. self.type = _type
  34. self.title = title
  35. self.videos = videos
  36. class YtVideo:
  37. """Stores the relevant attributes of a single youtube video."""
  38. def __init__(self, title, ytid):
  39. self.title = title
  40. self.video_id = ytid
  41. class Connector:
  42. """This class will retrieve all the necessary data from youtube using
  43. youtube-dl, thus bypassing the API.
  44. """
  45. def __init__(self, subscription_type, name, search=""):
  46. """Populates the object with all necessary data."""
  47. self._type = subscription_type
  48. self._name = name
  49. self._search = search
  50. self._conf = configuration.Conf.get_instance()
  51. self._title = ""
  52. self._url = ""
  53. self._construct_url()
  54. def _construct_url(self):
  55. """Constructs the URL to request from youtube-dl according to the
  56. subscription type and the given parameters.
  57. """
  58. if self._type == "user":
  59. self._url = "https://www.youtube.com/user/%s" \
  60. % urllib.parse.quote(self._name)
  61. elif self._type == "channel":
  62. self._url = "https://www.youtube.com/channel/%s" \
  63. % urllib.parse.quote(self._name)
  64. elif self._type == "search":
  65. self._url = "https://www.youtube.com/results?search_query=%s" \
  66. % urllib.parse.quote(self._search)
  67. elif self._type == "playlist":
  68. self._url = "https://www.youtube.com/playlist?list=%s" \
  69. % urllib.parse.quote(self._name)
  70. LOGGER.debug(_("Constructed URL for subscription: %s"), self._url)
  71. def _fetch_title(self):
  72. """Retrieves the title of the HTML page to use as a title for the
  73. subscription."""
  74. try:
  75. LOGGER.debug(_("Opening URL: %s to fetch title"), self._url)
  76. data = urllib.request.urlopen(self._url)
  77. except urllib.error.HTTPError as err:
  78. if err.code == 404 and self._type == "user":
  79. self._type = "channel"
  80. self._construct_url()
  81. try:
  82. data = urllib.request.urlopen(self._url)
  83. except urllib.error.HTTPError as exc:
  84. raise stov_exceptions.ChannelNotFound() from exc
  85. else:
  86. self._parse_title(data)
  87. else:
  88. raise stov_exceptions.ChannelNotFound()
  89. else:
  90. self._parse_title(data)
  91. def _parse_title(self, data):
  92. """ Parses the title from a HTML document
  93. :param data: HTTP connection to the document
  94. :type data: http.client.HTTPResponse
  95. """
  96. parsed_html = lxml.html.parse(data)
  97. data.close()
  98. i = 0
  99. for item in parsed_html.iter("title"):
  100. if i == 0:
  101. self._title = item.text_content().strip().replace("\n", "")
  102. i += 1
  103. if self._search != "" and self._type == "user":
  104. self._title += _(" search %s") % self._search
  105. def _fetch_videos(self, existing_videos):
  106. """Retrieves all the relevant videos in a subscription."""
  107. videos_list = []
  108. if self._type == "user" and self._search:
  109. video_ids = youtubedl_wrapper.get_ids(self._url)
  110. elif self._type == "playlist":
  111. video_ids = youtubedl_wrapper.get_ids(self._url, reverse=True)
  112. else:
  113. video_ids = youtubedl_wrapper.get_ids(self._url)
  114. LOGGER.debug("Got video IDs: %s", video_ids)
  115. if video_ids:
  116. for video_id in video_ids:
  117. video_exists = False
  118. if existing_videos:
  119. for existing_video in existing_videos:
  120. if video_id == existing_video.site_id:
  121. video_exists = True
  122. break
  123. if not video_exists:
  124. try:
  125. video_title = youtubedl_wrapper.get_title(
  126. self.construct_video_url(video_id))
  127. except subprocess.CalledProcessError as exc:
  128. raise stov_exceptions.YoutubeDlCallFailed() from exc
  129. else:
  130. videos_list.append(YtVideo(
  131. video_title,
  132. video_id))
  133. return videos_list
  134. def parse_api_data(self, existing_videos):
  135. """This method calls all necessary methods to retrieve the data
  136. and assembles them into a Channel object. The naming of this
  137. method was set according to the method in youtubeAPI to be
  138. compatible.
  139. """
  140. self._fetch_title()
  141. videos = self._fetch_videos(existing_videos)
  142. channel = YtChannel(self._type, self._title, videos)
  143. return channel
  144. @staticmethod
  145. def construct_video_url(ytid):
  146. """
  147. Resturns the URL to a specified youtube video
  148. :param ytid: Youtube ID of the video
  149. :type ytid: str
  150. :return: Video URL
  151. :rtype: str
  152. """
  153. url = "https://www.youtube.com/watch?v=%s" % ytid
  154. return url