yt: speed up retrieving feed data by starting multiple (up to 8, default is 4) http transfers in parallel

data/locale/deutsch.locale

@@ -1359,6 +1359,7 @@ moviebrowser.update_if_dest_empty_only Übernehmen nur wenn Ziel leer
 moviebrowser.use_dir Verzeichnis verwenden
 moviebrowser.use_movie_dir Wiedergabeverzeichnis verwenden
 moviebrowser.use_rec_dir Aufnahmeverzeichnis verwenden
+moviebrowser.yt_concurrent_connections Gleichzeitige Verbindungen 
 moviebrowser.yt_error Fehler beim laden des Youtube Feed
 moviebrowser.yt_max_results Max. Anzahl der zu holenden Feeds
 moviebrowser.yt_most_discussed Am meisten diskutiert

data/locale/english.locale

@@ -1359,6 +1359,7 @@ moviebrowser.update_if_dest_empty_only Copy if destination is empty only
 moviebrowser.use_dir Use directory
 moviebrowser.use_movie_dir Use movie directory
 moviebrowser.use_rec_dir Use record directory
+moviebrowser.yt_concurrent_connections Concurrent connections
 moviebrowser.yt_error Failed to load youtube feed
 moviebrowser.yt_max_results Max results to fetch
 moviebrowser.yt_most_discussed Most discussed

src/gui/moviebrowser.cpp

@@ -773,6 +773,7 @@ bool CMovieBrowser::loadSettings(MB_SETTINGS* settings)
 	settings->ytmode = configfile.getInt32("mb_ytmode", cYTFeedParser::MOST_POPULAR);
 	settings->ytresults = configfile.getInt32("mb_ytresults", 10);
 	settings->ytquality = configfile.getInt32("mb_ytquality", 22); // itag value (MP4, 720p)
+	settings->ytconcconn = configfile.getInt32("mb_ytconcconn", 4); // concurrent connections
 	settings->ytregion = configfile.getString("mb_ytregion", "default");
 	settings->ytsearch = configfile.getString("mb_ytsearch", "");
 	settings->ytvid = configfile.getString("mb_ytvid", "");
@@ -828,6 +829,7 @@ bool CMovieBrowser::saveSettings(MB_SETTINGS* settings)
 	configfile.setInt32("mb_ytmode", settings->ytmode);
 	configfile.setInt32("mb_ytresults", settings->ytresults);
 	configfile.setInt32("mb_ytquality", settings->ytquality);
+	configfile.setInt32("mb_ytconcconn", settings->ytconcconn);
 	configfile.setString("mb_ytregion", settings->ytregion);
 	configfile.setString("mb_ytsearch", settings->ytsearch);
 	configfile.setString("mb_ytvid", settings->ytvid);
@@ -3564,6 +3566,7 @@ void CMovieBrowser::loadYTitles(int mode, std::string search, std::string id)
 		ytparser.SetRegion(m_settings.ytregion);
 
 	ytparser.SetMaxResults(m_settings.ytresults);
+	ytparser.SetConcurrentDownloads(m_settings.ytconcconn);
 
 	if (!ytparser.Parsed() || (ytparser.GetFeedMode() != mode)) {
 		if (ytparser.ParseFeed((cYTFeedParser::yt_feed_mode_t)mode, search, id)) {
@@ -3695,8 +3698,10 @@ bool CMovieBrowser::showYTMenu()
 		{ 37, NONEXISTANT_LOCALE, "MP4 1080p"	 }
 	};
 	mainMenu.addItem(new CMenuOptionChooser(LOCALE_MOVIEBROWSER_YT_PREF_QUALITY, &m_settings.ytquality, YT_QUALITY_OPTIONS, YT_QUALITY_OPTION_COUNT, true, NULL, CRCInput::RC_nokey, "", true));
+	mainMenu.addItem(new CMenuOptionNumberChooser(LOCALE_MOVIEBROWSER_YT_CONCURRENT_CONNECTIONS, &m_settings.ytconcconn, true, 1, 8));
 
 	mainMenu.exec(NULL, "");
+	ytparser.SetConcurrentDownloads(m_settings.ytconcconn);
 	delete selector;
 
 	bool reload = false;

src/gui/moviebrowser.h

@@ -230,6 +230,7 @@ typedef struct
 	int ytmode;
 	int ytresults;
 	int ytquality;
+	int ytconcconn;
 	std::string ytregion;
 	std::string ytvid;
 	std::string ytsearch;

src/system/locals.h

@@ -1386,6 +1386,7 @@ typedef enum
 	LOCALE_MOVIEBROWSER_USE_DIR,
 	LOCALE_MOVIEBROWSER_USE_MOVIE_DIR,
 	LOCALE_MOVIEBROWSER_USE_REC_DIR,
+	LOCALE_MOVIEBROWSER_YT_CONCURRENT_CONNECTIONS,
 	LOCALE_MOVIEBROWSER_YT_ERROR,
 	LOCALE_MOVIEBROWSER_YT_MAX_RESULTS,
 	LOCALE_MOVIEBROWSER_YT_MOST_DISCUSSED,

src/system/locals_intern.h

@@ -1386,6 +1386,7 @@ const char * locale_real_names[] =
 	"moviebrowser.use_dir",
 	"moviebrowser.use_movie_dir",
 	"moviebrowser.use_rec_dir",
+	"moviebrowser.yt_concurrent_connections",
 	"moviebrowser.yt_error",
 	"moviebrowser.yt_max_results",
 	"moviebrowser.yt_most_discussed",

src/system/ytparser.cpp

@@ -22,6 +22,7 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
+#include <errno.h>
 
 #include <set>
 #include <map>
@@ -29,6 +30,9 @@
 #include <bitset>
 #include <string>
 
+#include <OpenThreads/ScopedLock>
+#include "set_threadname.h"
+
 #include "ytparser.h"
 
 #if LIBCURL_VERSION_NUM < 0x071507
@@ -90,6 +94,7 @@ cYTFeedParser::cYTFeedParser()
 	feedmode = -1;
 	tquality = "mqdefault";
 	max_results = 25;
+	concurrent_downloads = 2;
 	curl_handle = curl_easy_init();
 }
 
@@ -107,20 +112,23 @@ size_t cYTFeedParser::CurlWriteToString(void *ptr, size_t size, size_t nmemb, vo
         return size*nmemb;
 }
 
-bool cYTFeedParser::getUrl(std::string &url, std::string &answer)
+bool cYTFeedParser::getUrl(std::string &url, std::string &answer, CURL *_curl_handle)
 {
-	curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
+	if (!_curl_handle)
-	curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, &cYTFeedParser::CurlWriteToString);
+		_curl_handle = curl_handle;
-	curl_easy_setopt(curl_handle, CURLOPT_FILE, (void *)&answer);
+
-	curl_easy_setopt(curl_handle, CURLOPT_FAILONERROR, 1);
+	curl_easy_setopt(_curl_handle, CURLOPT_URL, url.c_str());
-	curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, URL_TIMEOUT);
+	curl_easy_setopt(_curl_handle, CURLOPT_WRITEFUNCTION, &cYTFeedParser::CurlWriteToString);
-	curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, (long)1);
+	curl_easy_setopt(_curl_handle, CURLOPT_FILE, (void *)&answer);
+	curl_easy_setopt(_curl_handle, CURLOPT_FAILONERROR, 1);
+	curl_easy_setopt(_curl_handle, CURLOPT_TIMEOUT, URL_TIMEOUT);
+	curl_easy_setopt(_curl_handle, CURLOPT_NOSIGNAL, (long)1);
 
 	char cerror[CURL_ERROR_SIZE];
-	curl_easy_setopt(curl_handle, CURLOPT_ERRORBUFFER, cerror);
+	curl_easy_setopt(_curl_handle, CURLOPT_ERRORBUFFER, cerror);
 
 	printf("try to get [%s] ...\n", url.c_str());
-	CURLcode httpres = curl_easy_perform(curl_handle);
+	CURLcode httpres = curl_easy_perform(_curl_handle);
 
 	printf("http: res %d size %d\n", httpres, answer.size());
 
@@ -131,31 +139,34 @@ bool cYTFeedParser::getUrl(std::string &url, std::string &answer)
 	return true;
 }
 
-bool cYTFeedParser::DownloadUrl(std::string &url, std::string &file)
+bool cYTFeedParser::DownloadUrl(std::string &url, std::string &file, CURL *_curl_handle)
 {
+	if (!_curl_handle)
+		_curl_handle = curl_handle;
+
 	FILE * fp = fopen(file.c_str(), "wb");
 	if (fp == NULL) {
 		perror(file.c_str());
 		return false;
 	}
-	curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
+	curl_easy_setopt(_curl_handle, CURLOPT_URL, url.c_str());
-	curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, NULL);
+	curl_easy_setopt(_curl_handle, CURLOPT_WRITEFUNCTION, NULL);
-	curl_easy_setopt(curl_handle, CURLOPT_FILE, fp);
+	curl_easy_setopt(_curl_handle, CURLOPT_FILE, fp);
-	curl_easy_setopt(curl_handle, CURLOPT_FAILONERROR, 1);
+	curl_easy_setopt(_curl_handle, CURLOPT_FAILONERROR, 1);
-	curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, URL_TIMEOUT);
+	curl_easy_setopt(_curl_handle, CURLOPT_TIMEOUT, URL_TIMEOUT);
-	curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, (long)1);
+	curl_easy_setopt(_curl_handle, CURLOPT_NOSIGNAL, (long)1);
 
 	char cerror[CURL_ERROR_SIZE];
-	curl_easy_setopt(curl_handle, CURLOPT_ERRORBUFFER, cerror);
+	curl_easy_setopt(_curl_handle, CURLOPT_ERRORBUFFER, cerror);
 
 	printf("try to get [%s] ...\n", url.c_str());
-	CURLcode httpres = curl_easy_perform(curl_handle);
+	CURLcode httpres = curl_easy_perform(_curl_handle);
 
 	double dsize;
-	curl_easy_getinfo(curl_handle, CURLINFO_SIZE_DOWNLOAD, &dsize);
+	curl_easy_getinfo(_curl_handle, CURLINFO_SIZE_DOWNLOAD, &dsize);
 	fclose(fp);
 
-	printf("http: res %d size %f.\n", httpres, dsize);
+	printf("http: res %d size %g.\n", httpres, dsize);
 
 	if (httpres != 0) {
 		printf("curl error: %s\n", cerror);
@@ -360,12 +371,22 @@ bool cYTFeedParser::parseFeedXml(std::string &answer)
 			/* save first one, if wanted not found */
 			if (vinfo.thumbnail.empty())
 				vinfo.thumbnail = thumbnail;
-			if (ParseVideoInfo(vinfo))
+			vinfo.ret = false;
 			videos.push_back(vinfo);
 		}
 		entry = entry->xmlNextNode;
 	}
 	xmlFreeDoc(answer_parser);
+
+	GetVideoUrls();
+
+	std::vector<cYTVideoInfo>::iterator pos = videos.begin();
+	while (pos != videos.end())
+		if ((*pos).ret)
+			++pos;
+		else
+			pos = videos.erase(pos);
+
 	parsed = !videos.empty();
 	return parsed;
 }
@@ -532,7 +553,7 @@ bool cYTFeedParser::ParseFeed(yt_feed_mode_t mode, std::string search, std::stri
 	return ParseFeed(url);
 }
 
-bool cYTFeedParser::ParseVideoInfo(cYTVideoInfo &vinfo)
+bool cYTFeedParser::ParseVideoInfo(cYTVideoInfo &vinfo, CURL *_curl_handle)
 {
 	bool ret = false;
 	std::vector<std::string> estr;
@@ -547,44 +568,98 @@ bool cYTFeedParser::ParseVideoInfo(cYTVideoInfo &vinfo)
 		vurl += "&ps=default&eurl=&gl=US&hl=en";
 		printf("cYTFeedParser::ParseVideoInfo: get [%s]\n", vurl.c_str());
 		std::string answer;
-		if (!getUrl(vurl, answer))
+		if (!getUrl(vurl, answer, _curl_handle))
 			continue;
 		ret = decodeVideoInfo(answer, vinfo);
 		if (ret)
 			break;
 	}
+	vinfo.ret = ret;
 	return ret;
 }
 
+void *cYTFeedParser::DownloadThumbnailsThread(void *arg)
+{
+	set_threadname("YT::DownloadThumbnails");
+	bool ret = true;
+	cYTFeedParser *caller = (cYTFeedParser *)arg;
+	CURL *c = curl_easy_init();
+	unsigned int i;
+	do {
+		OpenThreads::ScopedLock<OpenThreads::Mutex> m_lock(caller->mutex);
+		i = caller->worker_index++;
+	} while (i < caller->videos.size() && ((ret &= caller->DownloadThumbnail(caller->videos[i], c)) || true));
+	curl_easy_cleanup(c);
+	pthread_exit(&ret);
+}
+
+bool cYTFeedParser::DownloadThumbnail(cYTVideoInfo &vinfo, CURL *_curl_handle)
+{
+	if (!_curl_handle)
+		_curl_handle = curl_handle;
+	bool found = false;
+	if (!vinfo.thumbnail.empty()) {
+		std::string fname = thumbnail_dir + "/" + vinfo.id + ".jpg";
+		found = !access(fname.c_str(), F_OK);
+		if (!found)
+			found = DownloadUrl(vinfo.thumbnail, fname, _curl_handle);
+		if (found)
+			vinfo.tfile = fname;
+	}
+	return found;
+}
+
 bool cYTFeedParser::DownloadThumbnails()
 {
-	bool ret = false;
+	bool ret = true;
-	if (mkdir(thumbnail_dir.c_str(), 0755)) {
+	if (mkdir(thumbnail_dir.c_str(), 0755) && errno != EEXIST) {
 		perror(thumbnail_dir.c_str());
-		//return ret;
+		return false;
-	}
-	for (unsigned i = 0; i < videos.size(); i++) {
-		if (!videos[i].thumbnail.empty()) {
-			std::string fname = thumbnail_dir;
-			fname += "/";
-			fname += videos[i].id;
-			fname += ".jpg";
-			bool found = !access(fname.c_str(), F_OK);
-			if (!found)
-				found = DownloadUrl(videos[i].thumbnail, fname);
-			if (found)
-				videos[i].tfile = fname;
-			ret |= found;
 	}
+	unsigned int max_threads = concurrent_downloads;
+	if (videos.size() < max_threads)
+		max_threads = videos.size();
+	pthread_t ta[max_threads];
+	worker_index = 0;
+	for (unsigned i = 0; i < max_threads; i++)
+		pthread_create(&ta[i], NULL, cYTFeedParser::DownloadThumbnailsThread, this);
+	for (unsigned i = 0; i < max_threads; i++) {
+		void *r;
+		pthread_join(ta[i], &r);
+		ret &= *((bool *)r);
 	}
 	return ret;
 }
 
+void *cYTFeedParser::GetVideoUrlsThread(void *arg)
+{
+	set_threadname("YT::GetVideoUrls");
+	int ret = 0;
+	cYTFeedParser *caller = (cYTFeedParser *)arg;
+	CURL *c = curl_easy_init();
+	unsigned int i;
+	do {
+		OpenThreads::ScopedLock<OpenThreads::Mutex> m_lock(caller->mutex);
+		i = caller->worker_index++;
+	} while (i < caller->videos.size() && ((ret |= caller->ParseVideoInfo(caller->videos[i], c)) || true));
+	curl_easy_cleanup(c);
+	pthread_exit(&ret);
+}
+
 bool cYTFeedParser::GetVideoUrls()
 {
-	bool ret = false;
+	int ret = 0;
-	for (unsigned i = 0; i < videos.size(); i++) {
+	unsigned int max_threads = concurrent_downloads;
-		ret |= ParseVideoInfo(videos[i]);
+	if (videos.size() < max_threads)
+		max_threads = videos.size();
+	pthread_t ta[max_threads];
+	worker_index = 0;
+	for (unsigned i = 0; i < max_threads; i++)
+		pthread_create(&ta[i], NULL, cYTFeedParser::GetVideoUrlsThread, this);
+	for (unsigned i = 0; i < max_threads; i++) {
+		void *r;
+		pthread_join(ta[i], &r);
+		ret |= *((int *)r);
 	}
 	return ret;
 }

src/system/ytparser.h

@@ -28,6 +28,9 @@
 #include <map>
 #include <xmltree/xmlinterface.h>
 
+#include <OpenThreads/Thread>
+#include <OpenThreads/Condition>
+
 class cYTVideoUrl
 {
 	public:
@@ -56,6 +59,7 @@ class cYTVideoInfo
 		std::string published;
 		int duration;
 		yt_urlmap_t formats;
+		bool ret;
 
 		void Dump();
 		std::string GetUrl(int fmt = 0, bool mandatory = true);
@@ -80,6 +84,7 @@ class cYTFeedParser
 
 		int feedmode;
 		int max_results;
+		int concurrent_downloads;
 		bool parsed;
 		yt_video_list_t videos;
 
@@ -88,6 +93,10 @@ class cYTFeedParser
 		std::string getXmlData(xmlNodePtr node);
 
 		CURL *curl_handle;
+		OpenThreads::Mutex mutex;
+		int worker_index;
+		static void* GetVideoUrlsThread(void*);
+		static void* DownloadThumbnailsThread(void*);
 
 		static size_t CurlWriteToString(void *ptr, size_t size, size_t nmemb, void *data);
 		void encodeUrl(std::string &txt);
@@ -95,8 +104,8 @@ class cYTFeedParser
 		static void splitString(std::string &str, std::string delim, std::vector<std::string> &strlist, int start = 0);
 		static void splitString(std::string &str, std::string delim, std::map<std::string,std::string> &strmap, int start = 0);
 		static bool saveToFile(const char * name, std::string str);
-		bool getUrl(std::string &url, std::string &answer);
+		bool getUrl(std::string &url, std::string &answer, CURL *_curl_handle = NULL);
-		bool DownloadUrl(std::string &url, std::string &file);
+		bool DownloadUrl(std::string &url, std::string &file, CURL *_curl_handle = NULL);
 		bool parseFeedXml(std::string &answer);
 		bool decodeVideoInfo(std::string &answer, cYTVideoInfo &vinfo);
 		bool supportedFormat(int fmt);
@@ -124,7 +133,8 @@ class cYTFeedParser
 		~cYTFeedParser();
 
 		bool ParseFeed(yt_feed_mode_t mode = MOST_POPULAR, std::string search = "", std::string vid = "");
-		bool ParseVideoInfo(cYTVideoInfo &vinfo);
+		bool ParseVideoInfo(cYTVideoInfo &vinfo, CURL *_curl_handle = NULL);
+		bool DownloadThumbnail(cYTVideoInfo &vinfo, CURL *_curl_handle = NULL);
 		bool GetVideoUrls();
 		bool DownloadThumbnails();
 		void Dump();
@@ -140,6 +150,7 @@ class cYTFeedParser
 
 		void SetRegion(std::string reg) { region = reg; }
 		void SetMaxResults(int count) { max_results = count; }
+		void SetConcurrentDownloads(int count) { concurrent_downloads = count; }
 };
 
 #endif