diff --git a/data/locale/deutsch.locale b/data/locale/deutsch.locale index f16dfeaea..224ee5908 100644 --- a/data/locale/deutsch.locale +++ b/data/locale/deutsch.locale @@ -1359,6 +1359,7 @@ moviebrowser.update_if_dest_empty_only Übernehmen nur wenn Ziel leer moviebrowser.use_dir Verzeichnis verwenden moviebrowser.use_movie_dir Wiedergabeverzeichnis verwenden moviebrowser.use_rec_dir Aufnahmeverzeichnis verwenden +moviebrowser.yt_concurrent_connections Gleichzeitige Verbindungen moviebrowser.yt_error Fehler beim laden des Youtube Feed moviebrowser.yt_max_results Max. Anzahl der zu holenden Feeds moviebrowser.yt_most_discussed Am meisten diskutiert diff --git a/data/locale/english.locale b/data/locale/english.locale index 495d8bea3..73b50b659 100644 --- a/data/locale/english.locale +++ b/data/locale/english.locale @@ -1359,6 +1359,7 @@ moviebrowser.update_if_dest_empty_only Copy if destination is empty only moviebrowser.use_dir Use directory moviebrowser.use_movie_dir Use movie directory moviebrowser.use_rec_dir Use record directory +moviebrowser.yt_concurrent_connections Concurrent connections moviebrowser.yt_error Failed to load youtube feed moviebrowser.yt_max_results Max results to fetch moviebrowser.yt_most_discussed Most discussed diff --git a/src/gui/moviebrowser.cpp b/src/gui/moviebrowser.cpp index d0bff4bcd..fd2a30b09 100644 --- a/src/gui/moviebrowser.cpp +++ b/src/gui/moviebrowser.cpp @@ -773,6 +773,7 @@ bool CMovieBrowser::loadSettings(MB_SETTINGS* settings) settings->ytmode = configfile.getInt32("mb_ytmode", cYTFeedParser::MOST_POPULAR); settings->ytresults = configfile.getInt32("mb_ytresults", 10); settings->ytquality = configfile.getInt32("mb_ytquality", 22); // itag value (MP4, 720p) + settings->ytconcconn = configfile.getInt32("mb_ytconcconn", 4); // concurrent connections settings->ytregion = configfile.getString("mb_ytregion", "default"); settings->ytsearch = configfile.getString("mb_ytsearch", ""); settings->ytvid = configfile.getString("mb_ytvid", ""); @@ -828,6 +829,7 @@ bool CMovieBrowser::saveSettings(MB_SETTINGS* settings) configfile.setInt32("mb_ytmode", settings->ytmode); configfile.setInt32("mb_ytresults", settings->ytresults); configfile.setInt32("mb_ytquality", settings->ytquality); + configfile.setInt32("mb_ytconcconn", settings->ytconcconn); configfile.setString("mb_ytregion", settings->ytregion); configfile.setString("mb_ytsearch", settings->ytsearch); configfile.setString("mb_ytvid", settings->ytvid); @@ -3564,6 +3566,7 @@ void CMovieBrowser::loadYTitles(int mode, std::string search, std::string id) ytparser.SetRegion(m_settings.ytregion); ytparser.SetMaxResults(m_settings.ytresults); + ytparser.SetConcurrentDownloads(m_settings.ytconcconn); if (!ytparser.Parsed() || (ytparser.GetFeedMode() != mode)) { if (ytparser.ParseFeed((cYTFeedParser::yt_feed_mode_t)mode, search, id)) { @@ -3695,8 +3698,10 @@ bool CMovieBrowser::showYTMenu() { 37, NONEXISTANT_LOCALE, "MP4 1080p" } }; mainMenu.addItem(new CMenuOptionChooser(LOCALE_MOVIEBROWSER_YT_PREF_QUALITY, &m_settings.ytquality, YT_QUALITY_OPTIONS, YT_QUALITY_OPTION_COUNT, true, NULL, CRCInput::RC_nokey, "", true)); + mainMenu.addItem(new CMenuOptionNumberChooser(LOCALE_MOVIEBROWSER_YT_CONCURRENT_CONNECTIONS, &m_settings.ytconcconn, true, 1, 8)); mainMenu.exec(NULL, ""); + ytparser.SetConcurrentDownloads(m_settings.ytconcconn); delete selector; bool reload = false; diff --git a/src/gui/moviebrowser.h b/src/gui/moviebrowser.h index 015068236..188eb95b5 100644 --- a/src/gui/moviebrowser.h +++ b/src/gui/moviebrowser.h @@ -230,6 +230,7 @@ typedef struct int ytmode; int ytresults; int ytquality; + int ytconcconn; std::string ytregion; std::string ytvid; std::string ytsearch; diff --git a/src/system/locals.h b/src/system/locals.h index 36127b8f2..82a303ff3 100644 --- a/src/system/locals.h +++ b/src/system/locals.h @@ -1386,6 +1386,7 @@ typedef enum LOCALE_MOVIEBROWSER_USE_DIR, LOCALE_MOVIEBROWSER_USE_MOVIE_DIR, LOCALE_MOVIEBROWSER_USE_REC_DIR, + LOCALE_MOVIEBROWSER_YT_CONCURRENT_CONNECTIONS, LOCALE_MOVIEBROWSER_YT_ERROR, LOCALE_MOVIEBROWSER_YT_MAX_RESULTS, LOCALE_MOVIEBROWSER_YT_MOST_DISCUSSED, diff --git a/src/system/locals_intern.h b/src/system/locals_intern.h index fc0323ae9..66616dfe8 100644 --- a/src/system/locals_intern.h +++ b/src/system/locals_intern.h @@ -1386,6 +1386,7 @@ const char * locale_real_names[] = "moviebrowser.use_dir", "moviebrowser.use_movie_dir", "moviebrowser.use_rec_dir", + "moviebrowser.yt_concurrent_connections", "moviebrowser.yt_error", "moviebrowser.yt_max_results", "moviebrowser.yt_most_discussed", diff --git a/src/system/ytparser.cpp b/src/system/ytparser.cpp index 18455666a..29fce2986 100644 --- a/src/system/ytparser.cpp +++ b/src/system/ytparser.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,9 @@ #include #include +#include +#include "set_threadname.h" + #include "ytparser.h" #if LIBCURL_VERSION_NUM < 0x071507 @@ -90,6 +94,7 @@ cYTFeedParser::cYTFeedParser() feedmode = -1; tquality = "mqdefault"; max_results = 25; + concurrent_downloads = 2; curl_handle = curl_easy_init(); } @@ -107,20 +112,23 @@ size_t cYTFeedParser::CurlWriteToString(void *ptr, size_t size, size_t nmemb, vo return size*nmemb; } -bool cYTFeedParser::getUrl(std::string &url, std::string &answer) +bool cYTFeedParser::getUrl(std::string &url, std::string &answer, CURL *_curl_handle) { - curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, &cYTFeedParser::CurlWriteToString); - curl_easy_setopt(curl_handle, CURLOPT_FILE, (void *)&answer); - curl_easy_setopt(curl_handle, CURLOPT_FAILONERROR, 1); - curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, URL_TIMEOUT); - curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, (long)1); + if (!_curl_handle) + _curl_handle = curl_handle; + + curl_easy_setopt(_curl_handle, CURLOPT_URL, url.c_str()); + curl_easy_setopt(_curl_handle, CURLOPT_WRITEFUNCTION, &cYTFeedParser::CurlWriteToString); + curl_easy_setopt(_curl_handle, CURLOPT_FILE, (void *)&answer); + curl_easy_setopt(_curl_handle, CURLOPT_FAILONERROR, 1); + curl_easy_setopt(_curl_handle, CURLOPT_TIMEOUT, URL_TIMEOUT); + curl_easy_setopt(_curl_handle, CURLOPT_NOSIGNAL, (long)1); char cerror[CURL_ERROR_SIZE]; - curl_easy_setopt(curl_handle, CURLOPT_ERRORBUFFER, cerror); + curl_easy_setopt(_curl_handle, CURLOPT_ERRORBUFFER, cerror); printf("try to get [%s] ...\n", url.c_str()); - CURLcode httpres = curl_easy_perform(curl_handle); + CURLcode httpres = curl_easy_perform(_curl_handle); printf("http: res %d size %d\n", httpres, answer.size()); @@ -131,31 +139,34 @@ bool cYTFeedParser::getUrl(std::string &url, std::string &answer) return true; } -bool cYTFeedParser::DownloadUrl(std::string &url, std::string &file) +bool cYTFeedParser::DownloadUrl(std::string &url, std::string &file, CURL *_curl_handle) { + if (!_curl_handle) + _curl_handle = curl_handle; + FILE * fp = fopen(file.c_str(), "wb"); if (fp == NULL) { perror(file.c_str()); return false; } - curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, NULL); - curl_easy_setopt(curl_handle, CURLOPT_FILE, fp); - curl_easy_setopt(curl_handle, CURLOPT_FAILONERROR, 1); - curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, URL_TIMEOUT); - curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, (long)1); + curl_easy_setopt(_curl_handle, CURLOPT_URL, url.c_str()); + curl_easy_setopt(_curl_handle, CURLOPT_WRITEFUNCTION, NULL); + curl_easy_setopt(_curl_handle, CURLOPT_FILE, fp); + curl_easy_setopt(_curl_handle, CURLOPT_FAILONERROR, 1); + curl_easy_setopt(_curl_handle, CURLOPT_TIMEOUT, URL_TIMEOUT); + curl_easy_setopt(_curl_handle, CURLOPT_NOSIGNAL, (long)1); char cerror[CURL_ERROR_SIZE]; - curl_easy_setopt(curl_handle, CURLOPT_ERRORBUFFER, cerror); + curl_easy_setopt(_curl_handle, CURLOPT_ERRORBUFFER, cerror); printf("try to get [%s] ...\n", url.c_str()); - CURLcode httpres = curl_easy_perform(curl_handle); + CURLcode httpres = curl_easy_perform(_curl_handle); double dsize; - curl_easy_getinfo(curl_handle, CURLINFO_SIZE_DOWNLOAD, &dsize); + curl_easy_getinfo(_curl_handle, CURLINFO_SIZE_DOWNLOAD, &dsize); fclose(fp); - printf("http: res %d size %f.\n", httpres, dsize); + printf("http: res %d size %g.\n", httpres, dsize); if (httpres != 0) { printf("curl error: %s\n", cerror); @@ -360,12 +371,22 @@ bool cYTFeedParser::parseFeedXml(std::string &answer) /* save first one, if wanted not found */ if (vinfo.thumbnail.empty()) vinfo.thumbnail = thumbnail; - if (ParseVideoInfo(vinfo)) - videos.push_back(vinfo); + vinfo.ret = false; + videos.push_back(vinfo); } entry = entry->xmlNextNode; } xmlFreeDoc(answer_parser); + + GetVideoUrls(); + + std::vector::iterator pos = videos.begin(); + while (pos != videos.end()) + if ((*pos).ret) + ++pos; + else + pos = videos.erase(pos); + parsed = !videos.empty(); return parsed; } @@ -532,7 +553,7 @@ bool cYTFeedParser::ParseFeed(yt_feed_mode_t mode, std::string search, std::stri return ParseFeed(url); } -bool cYTFeedParser::ParseVideoInfo(cYTVideoInfo &vinfo) +bool cYTFeedParser::ParseVideoInfo(cYTVideoInfo &vinfo, CURL *_curl_handle) { bool ret = false; std::vector estr; @@ -547,44 +568,98 @@ bool cYTFeedParser::ParseVideoInfo(cYTVideoInfo &vinfo) vurl += "&ps=default&eurl=&gl=US&hl=en"; printf("cYTFeedParser::ParseVideoInfo: get [%s]\n", vurl.c_str()); std::string answer; - if (!getUrl(vurl, answer)) + if (!getUrl(vurl, answer, _curl_handle)) continue; ret = decodeVideoInfo(answer, vinfo); if (ret) break; } + vinfo.ret = ret; return ret; } +void *cYTFeedParser::DownloadThumbnailsThread(void *arg) +{ + set_threadname("YT::DownloadThumbnails"); + bool ret = true; + cYTFeedParser *caller = (cYTFeedParser *)arg; + CURL *c = curl_easy_init(); + unsigned int i; + do { + OpenThreads::ScopedLock m_lock(caller->mutex); + i = caller->worker_index++; + } while (i < caller->videos.size() && ((ret &= caller->DownloadThumbnail(caller->videos[i], c)) || true)); + curl_easy_cleanup(c); + pthread_exit(&ret); +} + +bool cYTFeedParser::DownloadThumbnail(cYTVideoInfo &vinfo, CURL *_curl_handle) +{ + if (!_curl_handle) + _curl_handle = curl_handle; + bool found = false; + if (!vinfo.thumbnail.empty()) { + std::string fname = thumbnail_dir + "/" + vinfo.id + ".jpg"; + found = !access(fname.c_str(), F_OK); + if (!found) + found = DownloadUrl(vinfo.thumbnail, fname, _curl_handle); + if (found) + vinfo.tfile = fname; + } + return found; +} + bool cYTFeedParser::DownloadThumbnails() { - bool ret = false; - if (mkdir(thumbnail_dir.c_str(), 0755)) { + bool ret = true; + if (mkdir(thumbnail_dir.c_str(), 0755) && errno != EEXIST) { perror(thumbnail_dir.c_str()); - //return ret; + return false; } - for (unsigned i = 0; i < videos.size(); i++) { - if (!videos[i].thumbnail.empty()) { - std::string fname = thumbnail_dir; - fname += "/"; - fname += videos[i].id; - fname += ".jpg"; - bool found = !access(fname.c_str(), F_OK); - if (!found) - found = DownloadUrl(videos[i].thumbnail, fname); - if (found) - videos[i].tfile = fname; - ret |= found; - } + unsigned int max_threads = concurrent_downloads; + if (videos.size() < max_threads) + max_threads = videos.size(); + pthread_t ta[max_threads]; + worker_index = 0; + for (unsigned i = 0; i < max_threads; i++) + pthread_create(&ta[i], NULL, cYTFeedParser::DownloadThumbnailsThread, this); + for (unsigned i = 0; i < max_threads; i++) { + void *r; + pthread_join(ta[i], &r); + ret &= *((bool *)r); } return ret; } +void *cYTFeedParser::GetVideoUrlsThread(void *arg) +{ + set_threadname("YT::GetVideoUrls"); + int ret = 0; + cYTFeedParser *caller = (cYTFeedParser *)arg; + CURL *c = curl_easy_init(); + unsigned int i; + do { + OpenThreads::ScopedLock m_lock(caller->mutex); + i = caller->worker_index++; + } while (i < caller->videos.size() && ((ret |= caller->ParseVideoInfo(caller->videos[i], c)) || true)); + curl_easy_cleanup(c); + pthread_exit(&ret); +} + bool cYTFeedParser::GetVideoUrls() { - bool ret = false; - for (unsigned i = 0; i < videos.size(); i++) { - ret |= ParseVideoInfo(videos[i]); + int ret = 0; + unsigned int max_threads = concurrent_downloads; + if (videos.size() < max_threads) + max_threads = videos.size(); + pthread_t ta[max_threads]; + worker_index = 0; + for (unsigned i = 0; i < max_threads; i++) + pthread_create(&ta[i], NULL, cYTFeedParser::GetVideoUrlsThread, this); + for (unsigned i = 0; i < max_threads; i++) { + void *r; + pthread_join(ta[i], &r); + ret |= *((int *)r); } return ret; } diff --git a/src/system/ytparser.h b/src/system/ytparser.h index 980525f5d..fb97d65d1 100644 --- a/src/system/ytparser.h +++ b/src/system/ytparser.h @@ -28,6 +28,9 @@ #include #include +#include +#include + class cYTVideoUrl { public: @@ -56,6 +59,7 @@ class cYTVideoInfo std::string published; int duration; yt_urlmap_t formats; + bool ret; void Dump(); std::string GetUrl(int fmt = 0, bool mandatory = true); @@ -80,6 +84,7 @@ class cYTFeedParser int feedmode; int max_results; + int concurrent_downloads; bool parsed; yt_video_list_t videos; @@ -88,6 +93,10 @@ class cYTFeedParser std::string getXmlData(xmlNodePtr node); CURL *curl_handle; + OpenThreads::Mutex mutex; + int worker_index; + static void* GetVideoUrlsThread(void*); + static void* DownloadThumbnailsThread(void*); static size_t CurlWriteToString(void *ptr, size_t size, size_t nmemb, void *data); void encodeUrl(std::string &txt); @@ -95,8 +104,8 @@ class cYTFeedParser static void splitString(std::string &str, std::string delim, std::vector &strlist, int start = 0); static void splitString(std::string &str, std::string delim, std::map &strmap, int start = 0); static bool saveToFile(const char * name, std::string str); - bool getUrl(std::string &url, std::string &answer); - bool DownloadUrl(std::string &url, std::string &file); + bool getUrl(std::string &url, std::string &answer, CURL *_curl_handle = NULL); + bool DownloadUrl(std::string &url, std::string &file, CURL *_curl_handle = NULL); bool parseFeedXml(std::string &answer); bool decodeVideoInfo(std::string &answer, cYTVideoInfo &vinfo); bool supportedFormat(int fmt); @@ -124,7 +133,8 @@ class cYTFeedParser ~cYTFeedParser(); bool ParseFeed(yt_feed_mode_t mode = MOST_POPULAR, std::string search = "", std::string vid = ""); - bool ParseVideoInfo(cYTVideoInfo &vinfo); + bool ParseVideoInfo(cYTVideoInfo &vinfo, CURL *_curl_handle = NULL); + bool DownloadThumbnail(cYTVideoInfo &vinfo, CURL *_curl_handle = NULL); bool GetVideoUrls(); bool DownloadThumbnails(); void Dump(); @@ -140,6 +150,7 @@ class cYTFeedParser void SetRegion(std::string reg) { region = reg; } void SetMaxResults(int count) { max_results = count; } + void SetConcurrentDownloads(int count) { concurrent_downloads = count; } }; #endif