Add maxfails option and mark videos as permanently failed once the defined value is reached

CHANGELOG

@@ -4,6 +4,8 @@
 * Added notify option
 * Added "verbose" and "quiet" output modes
 * Added switch to clean the database of old video entries
+* Added maxfails option and mark videos as permanently failed when that value is reached
+* Implemented MAXVIDEOS configuration option to limit the number of videos retrieved from the youtube API
 * Changed --lsvids to print the download status
 * Made the youtube-dl output shown to the user depend on the defined output level
 * Now uses youtube API version 2

README

@@ -40,6 +40,8 @@ VIDEOCODEC: Video codec used for downloaded videos (valid values: h264, webm or
 MAXRESOLUTION: Maximum resolution to use for downloaded videos, please note
 	that not all codecs and resolutions might be available so a lower resolution
 	or other codec might be used in these cases
+MAXFAILS: Number of times the download of a video may fail until it is marked as failed
+	and won't be tried again
 
 === KNOWN ISSUES ===
 

TODO

@@ -4,4 +4,3 @@ Long term goals:
 * Add full german translation
 * Add interactive configuration option
 * Add additional video sites
-* Improve handling of situations where a video is permanently unavailable

configuration.py

@@ -44,10 +44,11 @@ class conf(object):
 				"password": "",
 				"youtube-dl": "",
 				"notify": "yes",
-				"config_version": "5",
-				"db_version": "1",
+				"config_version": "6",
+				"db_version": "2",
 				"videocodec": "h264",
-				"maxresolution": "1080p"
+				"maxresolution": "1080p",
+				"maxfails": "50"
 				}
 		self.dbpath = str(os.environ['HOME']) + "/.stov/" + self.values["database"]
 		self.outputlevel = "default"
@@ -101,7 +102,8 @@ class conf(object):
 						description TEXT,
 						ytid TEXT,
 						subscription_id INTEGER,
-						downloaded int
+						downloaded INT,
+						failcnt INT DEFAULT 0
 						);""")
 			self.__database.commit()
 			self.__database.close()
@@ -162,6 +164,7 @@ class conf(object):
 			printf(_("Invalid config version read"), outputlevel="default",
 				level=self.outputlevel, descriptor="stderr")
 		self.values["config_version"] = "0"
+		self.__currentdbversion = self.values["db_version"]
 		self.ReadConfig()
 		printf(_("Found running version: " + self.values["config_version"] + "\n" +
 			"Current version: " + str(self.__currentversion)),
@@ -185,18 +188,13 @@ class conf(object):
 		"""Checks the database if it is up-to-date"""
 		printf(_("Checking current and running database version."),
 			outputlevel="verbose", level=self.outputlevel, descriptor="stderr")
-		try:
-			self.__currentversion = int(self.values["db_version"])
-		except ValueError:
-			printf(_("Invalid config version read"), outputlevel="default",
-				level=self.outputlevel, descriptor="stderr")
 		self.values["db_version"] = "0"
 		self.ReadConfig()
 		printf(_("Found running database version: " + self.values["db_version"] +
 			"\n" + "Current version: " + str(self.__currentversion)),
 			outputlevel="verbose", level=self.outputlevel, descriptor="stderr")
-		if self.values["db_version"] == "0" \
-		or int(self.values["db_version"]) < self.__currentversion:
+		if self.values["db_version"] == "0" or \
+		int(self.values["db_version"]) < int(self.__currentdbversion):
 			self.values["db_version"] = str(self.__currentversion)
 			return False
 		else:
@@ -205,7 +203,23 @@ class conf(object):
 
 	def UpdateDB(self):
 		"""Performs database changes that need to be done"""
+		self.ReadConfig()
 		if int(self.values["db_version"]) == 1:
+			try:
+				self.__database = sqlite3.connect(self.dbpath)
+			except sqlite3.OperationalError:
+				printf(_("The database could not be updated, please "
+					"check that the configuration directory exists and is writable"),
+					outputlevel = "default", level=self.outputlevel, descriptor="stderr")
+			else:
+				self.__dbcursor = self.__database.cursor()
+				self.__dbcursor.execute("ALTER TABLE videos add column failcnt int DEFAULT 0;")
+				self.__database.commit()
+				self.__database.close()
+				self.ReadConfig()
+				self.values["db_version"] = "2"
+				self.WriteConfig()
+		else:
 			pass
 
 	def GetYoutubeParameter(self):

stov.py

@@ -358,7 +358,7 @@ FROM subscriptions")
 		subscriptions = cursor.fetchall()
 	itag_value = conf.GetYoutubeParameter()
 	if itag_value == 0:
-		printf(_("Codec and resolution could not be determined, using maximum"
+		printf(_("Codec and resolution could not be determined, using maximum "
 			"possible value"), outputlevel="verbose",
 			level=conf.outputlevel, descriptor="stderr")
 		itag_value = 38
@@ -436,8 +436,8 @@ FROM subscriptions")
 			for i in mailcontent:
 				printf(i, outputlevel="default", level=conf.outputlevel, descriptor="stdout")
 	else:
-		printf(_("Could not determine how you want to be informed"
-					"about new videos, please check the notify parameter"
+		printf(_("Could not determine how you want to be informed "
+					"about new videos, please check the notify parameter "
 					"in your configuration"), outputlevel="default",
 					level=conf.outputlevel, descriptor="stderr")
 

subscription.py

@@ -121,7 +121,7 @@ class sub(object):
 			% self.__title), outputlevel="verbose", level=self.__conf.outputlevel,
 			descriptor="stderr")
 		self.__videoquerybysubscription = "SELECT id, title, description, \
-		ytid, downloaded FROM videos WHERE subscription_id=?"
+		ytid, downloaded, failcnt FROM videos WHERE subscription_id=?"
 		self.__cursor.execute(self.__videoquerybysubscription, (self.__ID,))
 		self.__videodata = self.__cursor.fetchall()
 		for i in self.__videodata:
@@ -129,7 +129,7 @@ class sub(object):
 				outputlevel="verbose", level=self.__conf.outputlevel, descriptor="stderr")
 			self.__videos.append(youtube.video(id=i[0],
 			title=i[1], description=i[2], ytid=i[3],
-			downloaded=i[4], conf=self.__conf))
+			downloaded=i[4], failcount=i[5], conf=self.__conf))
 
 		self.__connection.close()
 
@@ -157,6 +157,9 @@ class sub(object):
 			elif i.downloaded == 1:
 				printf(i.title + _("   (downloaded)"), outputlevel="default",
 					level=self.__conf.outputlevel, descriptor="stdout")
+			elif i.downloaded == -1:
+				printf(i.title + _("   (failed)"), outputlevel="default",
+				level=self.__conf.outputlevel, descriptor="stdout")
 
 	def AddSub(self):
 		"""Adds a new subscription to the database"""

youtube.py

@@ -25,13 +25,14 @@ import subprocess
 from outputhelper import printf
 
 class video(object):
-	def __init__(self, title, description, ytid, conf, downloaded, id=0):
+	def __init__(self, title, description, ytid, conf, downloaded, failcount = 0, id=0):
 		self.__ID = id
 		self.title = title
 		self.description = description
 		self.ytid = ytid
 		self.__conf = conf
 		self.downloaded = downloaded
+		self.failcnt = int(failcount)
 
 	def DownloadVideo(self, directory, itag_value):
 		"""Downloads the video by calling youtube-dl as an external process"""
@@ -85,6 +86,26 @@ and can't be created. Please check your configuration and try again"),
 							"due to some problem with youtube-dl. If this happens to more than one "
 							"video, please check your youtube-dl version" % self.title),
 					outputlevel="default", level=self.__conf.outputlevel, descriptor="stderr")
+				try:
+					self.__database = sqlite3.connect(self.__conf.dbpath)
+				except sqlite3.OperationalError:
+					printf(_("The Video \"%s\" has failed "
+								"downloading but the status could not be updated in the database. "
+								"Please check what went wrong and correct it" % self.title),
+						outputlevel="default", level=self.__conf.outputlevel, descriptor="stderr")
+				else:
+					self.__cursor = self.__database.cursor()
+					self.failcnt = int(self.failcnt) + 1
+					if self.failcnt >= int(self.__conf.values["maxfails"]):
+						printf(_("The video \"%s\" has failed downloading too often, marking as"
+							" failed" % self.title), outputlevel="default", level=self.__conf.outputlevel,
+							descriptor="stderr")
+						self.__statement = "UPDATE videos SET downloaded = -1 WHERE id = ?"
+						self.__cursor.execute(self.__statement, (self.__ID,))
+					self.__statement = "UPDATE videos SET failcnt = ? WHERE id = ?"
+					self.__cursor.execute(self.__statement, (self.failcnt, self.__ID))
+					self.__database.commit()
+					self.__database.close()
 				return False
 			else:
 				return True