implemented the noapi Connector class to retrieve all necessary data using only youtube-dl without needing the API, updated TODO

TODO

@@ -2,5 +2,7 @@ TODOs for 1.0:
 
 * Reintroduce some of the removed debug output
 * Test the whole code again
+* Update German translation
 * Remove python 2 compatibility code, support python 3 only
 * Implement youtube-dl as alternative mode to API version 2
+* Check with pep8 and pylinth to do coding style improvements before the next release

lib_stov/configuration.py

@@ -48,12 +48,13 @@ class conf(object):
             "password": "",
             "youtube-dl": "",
             "notify": "yes",
-            "config_version": "7",
+            "config_version": "8",
             "db_version": "3",
             "videocodec": "h264",
             "maxresolution": "1080p",
             "maxfails": 50,
-            "check_title": "no"
+            "check_title": "no",
+            "use_api": True
         }
 
         self.__explanations = {
@@ -145,7 +146,7 @@ class conf(object):
         except IOError:
             raise stov_exceptions.ConfigFileReadErrorException()
         else:
-            self.values = json.load(configfile)
+            self.values.update(json.load(configfile))
             configfile.close()
 
     def CheckConfig(self):

lib_stov/noapi.py

@@ -0,0 +1,117 @@
+#
+#        This file is part of stov, written by Helmut Pozimski 2012-2014.
+#
+#       stov is free software: you can redistribute it and/or modify
+#       it under the terms of the GNU General Public License as published by
+#       the Free Software Foundation, version 2 of the License.
+#
+#       stov is distributed in the hope that it will be useful,
+#       but WITHOUT ANY WARRANTY; without even the implied warranty of
+#       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#       GNU General Public License for more details.
+#
+#       You should have received a copy of the GNU General Public License
+#       along with stov.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# -*- coding: utf8 -*-
+
+import subprocess
+import sys
+import lxml.html
+
+if sys.version_info >= (3,):
+    import urllib.request as urllib2
+else:
+    import urllib2
+
+from lib_stov import youtubeAPI
+from lib_stov import stov_exceptions
+
+class Connector(object):
+    """This class will retrieve all the necessary data from youtube using
+    youtube-dl, thus bypassing the API.
+    """
+    def __init__(self, type, name, conf, search = ""):
+        """Populates the object with all necessary data."""
+        self._type = type
+        self._name = name
+        self._search = search
+        self._conf = conf
+        self._title = ""
+        self._url = ""
+        self._construct_url()
+
+    def _construct_url(self):
+        if self._type == "channel":
+            self._url = "https://www.youtube.com/user/%s" \
+                        % urllib2.quote(self._name)
+        elif self._type == "search":
+            self._url = "https://www.youtube.com/results?search_query=%s"\
+                        % urllib2.quote(self._search)
+        elif self._type == "playlist":
+            self._url = "https://www.youtube.com/playlist?list=%s" \
+                        % urllib2.quote(self._search)
+
+    def _fetch_title(self):
+        """Retrieves the title of the HTML page to use as a title for the
+        subscription."""
+        data = urllib2.urlopen(self._url)
+        parsed_html = lxml.html.parse(data)
+        i = 0
+        for item in parsed_html.iter("title"):
+            if i == 0:
+                self._title = item.text_content().strip().replace("\n", "")
+            i += 1
+
+    def _fetch_videos(self):
+        """Retrieves all the relevant videos in a subscription."""
+        videos_list = []
+        if self._type == "channel" and self._search != "":
+            try:
+                video_ids = subprocess.check_output([self._conf.values["youtube-dl"], "--max-downloads",
+                                                                   self._conf.values["maxvideos"],
+                                                                   "--match-title",
+                                                                   self._search,
+                                                                   "--get-id",
+                                                                   self._url])
+            except subprocess.CalledProcessError as e:
+                video_ids = e.output.strip()
+        else:
+            try:
+                video_ids = subprocess.check_output([self._conf.values["youtube-dl"], "--max-downloads",
+                                                                   self._conf.values["maxvideos"],
+                                                                   "--get-id",
+                                                                   self._url])
+            except subprocess.CalledProcessError as e:
+                video_ids = e.output.strip()
+        for video_id in video_ids.split("\n"):
+            try:
+                video_title = subprocess.check_output([
+                self._conf.values["youtube-dl"], "--get-title",
+                "https://www.youtube.com/watch?v=%s"
+                % video_id]).strip()
+                video_description = subprocess.check_output([
+                    self._conf.values["youtube-dl"], "--get-description",
+                    "https://www.youtube.com/watch?v=%s"
+                    % video_id]).strip()
+            except subprocess.CalledProcessError:
+                raise stov_exceptions.YoutubeDlCallFailed()
+            else:
+                videos_list.append(youtubeAPI.YtVideo(video_title,
+                                                        video_description,
+                                                        video_id))
+        return videos_list
+
+    def ParseAPIData(self):
+        """This method calls all necessary methods to retrieve the data
+                and assembles them into a Channel object. The naming of this
+                method is set according to the method in youtubeAPI to be
+                compatible.
+        """
+        self._fetch_title()
+        videos = self._fetch_videos()
+        channel = youtubeAPI.YtChannel()
+        channel.title = self._title
+        channel.videos = videos
+        return channel

lib_stov/stov_exceptions.py

@@ -165,6 +165,7 @@ class DownloadDirectoryCreationFailedException(Exception):
     def __str__(self):
         return self.__message
 
+
 class ConfigurationMigrationFailed(Exception):
     """This exception will be raised when the migration of the configuration
     to the json format fails.
@@ -174,4 +175,14 @@ class ConfigurationMigrationFailed(Exception):
                           "format failed.")
 
     def __str__(self):
-        return  self._message
+        return  self._message
+
+class YoutubeDlCallFailed(Exception):
+    """This exception will be raised when a call to youtube-dl fails because
+        of an error returned by youtube-dl.
+    """
+    def __init__(self):
+        self._message = _("Exectution of youtube-dl failed.")
+
+    def __str__(self):
+        return self._message

lib_stov/subscription.py

@@ -21,6 +21,7 @@ from __future__ import unicode_literals
 from lib_stov import youtubeAPI
 from lib_stov import youtube
 from lib_stov import stov_exceptions
+from lib_stov import noapi
 
 
 class sub(object):
@@ -46,7 +47,12 @@ class sub(object):
         elif int(disabled) == 1:
             self.__disabled = True
 
-        self._connector = youtubeAPI.Connector(self.__type, self.__name, self.__search, self.__conf)
+        if self.__conf.values["use_api"]:
+            self._connector = youtubeAPI.Connector(self.__type, self.__name,
+                                                   self.__search, self.__conf)
+        else:
+            self._connector = noapi.Connector(self.__type, self.__name,
+                                              self.__conf, self.__search)
 
     def GetTitle(self):
         return self.__title

lib_stov/youtubeAPI.py

@@ -27,7 +27,8 @@ else:
     import urllib2
 
 from xml.dom.minidom import parseString
-from lib_stov import stov_exceptions
+import stov_exceptions
+
 
 class Parser(object):
     def __init__(self, api_result):
@@ -106,7 +107,7 @@ class Connector(object):
         except urllib2.HTTPError:
                 raise stov_exceptions.NoDataFromYoutubeAPIException()
         except ssl.SSLError:
-                raise stov_exceptions.YoutubeAPITimeoutException(self._subscription.title)
+                raise stov_exceptions.YoutubeAPITimeoutException(self._name)
         else:
             parser = Parser(APIResponse)
             parsed_response = parser.parse()

stov

@@ -39,6 +39,7 @@ from lib_stov import subscription
 from lib_stov import configuration
 from lib_stov import stov_exceptions
 from lib_stov import database
+from lib_stov import noapi
 
 
 """Setup the logger to log messages to stdout and stderr"""