From c2575f6261c143623962c362e241147d28977978 Mon Sep 17 00:00:00 2001 From: Breezyslasher <167659924+Breezyslasher@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:38:50 -0400 Subject: [PATCH 1/4] Add files via upload --- addon.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/addon.xml b/addon.xml index 0b6b39d58..3cdb11aa0 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + @@ -86,7 +86,9 @@ https://forum.kodi.tv/showthread.php?tid=329767 https://github.com/CastagnaIT/plugin.video.netflix - v1.23.5 (2025-08-24) + v1.23.6 (2026-08-25) +- Fix login with E-Mail/Password, the sign in is confirmed with the code sent by Netflix +v1.23.5 (2025-08-24) - Fix esn error on login due to website changes - Fix Nonetype error on startup due to website changes From 08bbf46d8fde1cfa22333dadfcb7b721d42727b7 Mon Sep 17 00:00:00 2001 From: Breezyslasher <167659924+Breezyslasher@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:41:13 -0400 Subject: [PATCH 2/4] Add files via upload --- resources/settings.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/resources/settings.xml b/resources/settings.xml index 3f889c214..8f588bcb0 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -557,11 +557,6 @@ true - - 0 - true - - 0 true From c153cafd866556b174f478b1475ef53a4310eb22 Mon Sep 17 00:00:00 2001 From: Breezyslasher <167659924+Breezyslasher@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:43:48 -0400 Subject: [PATCH 3/4] Add files via upload --- resources/lib/common/ipc.py | 16 +- resources/lib/common/pathops.py | 4 +- resources/lib/globals.py | 32 +- resources/lib/kodi/infolabels.py | 231 +- resources/lib/navigation/actions.py | 79 +- resources/lib/navigation/directory.py | 31 +- resources/lib/navigation/directory_search.py | 16 +- .../nfsession/directorybuilder/dir_builder.py | 259 +- .../directorybuilder/dir_builder_items.py | 29 +- .../directorybuilder/dir_path_requests.py | 2175 ++++++++++++++++- .../lib/services/nfsession/msl/profiles.py | 2 +- .../lib/services/nfsession/nfsession_ops.py | 207 +- .../lib/services/nfsession/session/access.py | 425 +++- .../services/nfsession/session/endpoints.py | 7 - .../nfsession/session/http_requests.py | 103 +- .../services/playback/action_controller.py | 31 +- .../lib/services/playback/am_playback.py | 13 +- .../lib/services/playback/am_video_events.py | 39 +- resources/lib/utils/api_paths.py | 44 +- resources/lib/utils/api_requests.py | 67 +- resources/lib/utils/data_types.py | 17 +- resources/lib/utils/website.py | 68 +- 22 files changed, 3556 insertions(+), 339 deletions(-) diff --git a/resources/lib/common/ipc.py b/resources/lib/common/ipc.py index efb20f26a..8da523778 100644 --- a/resources/lib/common/ipc.py +++ b/resources/lib/common/ipc.py @@ -18,6 +18,8 @@ from .misc_utils import run_threaded IPC_TIMEOUT_SECS = 20 +# The login can wait the user that gets the one-time code sent by Netflix +IPC_TIMEOUT_SECS_LOGIN = 600 # IPC over HTTP endpoints IPC_ENDPOINT_CACHE = '/netflix_service/cache' @@ -75,7 +77,7 @@ def _send_signal(signal, data): @measure_exec_time_decorator() -def make_call(func_name, data=None, endpoint=IPC_ENDPOINT_NFSESSION): +def make_call(func_name, data=None, endpoint=IPC_ENDPOINT_NFSESSION, timeout=IPC_TIMEOUT_SECS): """ Make an IPC call :param func_name: function name @@ -91,11 +93,11 @@ def make_call(func_name, data=None, endpoint=IPC_ENDPOINT_NFSESSION): # https://github.com/xbmc/xbmc/issues/19332 # https://github.com/CastagnaIT/script.module.addon.connector if G.IPC_OVER_HTTP: - return make_http_call(endpoint, func_name, data) - return make_addonsignals_call(func_name, data) + return make_http_call(endpoint, func_name, data, timeout) + return make_addonsignals_call(func_name, data, timeout) -def make_http_call(endpoint, func_name, data=None): +def make_http_call(endpoint, func_name, data=None, timeout=IPC_TIMEOUT_SECS): """ Make an IPC call via HTTP and wait for it to return. The contents of data will be expanded to kwargs and passed into the target function. @@ -109,7 +111,7 @@ def make_http_call(endpoint, func_name, data=None): try: with urlopen(url=url, data=pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL), - timeout=IPC_TIMEOUT_SECS) as f: + timeout=timeout) as f: received_data = f.read() if received_data: _data = pickle.loads(received_data) @@ -127,7 +129,7 @@ def make_http_call(endpoint, func_name, data=None): raise exceptions.BackendNotReady(err_msg) from exc -def make_addonsignals_call(callname, data): +def make_addonsignals_call(callname, data, timeout=IPC_TIMEOUT_SECS): """ Make an IPC call via AddonSignals and wait for it to return. The contents of data will be expanded to kwargs and passed into the target function. @@ -138,7 +140,7 @@ def make_addonsignals_call(callname, data): source_id=G.ADDON_ID, signal=callname, data=_data, - timeout_ms=IPC_TIMEOUT_SECS * 1000, + timeout_ms=timeout * 1000, use_timeout_exception=True) _result = pickle.loads(b64decode(result)) if isinstance(_result, Exception): diff --git a/resources/lib/common/pathops.py b/resources/lib/common/pathops.py index befa51c63..8f3487ac6 100644 --- a/resources/lib/common/pathops.py +++ b/resources/lib/common/pathops.py @@ -22,10 +22,10 @@ def get_path(path, search_space, include_key=False): def get_path_safe(path, search_space, include_key=False, default=None): """Retrieve a value from a nested dict by following the path. - Returns default if any key in the path does not exist.""" + Returns default if the path is missing or cannot be traversed.""" try: return get_path(path, search_space, include_key) - except (KeyError, IndexError): + except (KeyError, IndexError, TypeError): return default diff --git a/resources/lib/globals.py b/resources/lib/globals.py index 261c41a62..4ee3d37c7 100644 --- a/resources/lib/globals.py +++ b/resources/lib/globals.py @@ -73,10 +73,16 @@ class GlobalVariables: 'request_context_name': 'mylist', 'view': VIEW_MYLIST, 'has_sort_setting': True, - 'query_without_reference': True}), + 'query_without_reference': True, + 'label_id': 30167, + 'description_id': None, + 'icon': 'DefaultVideoPlaylists.png'}), ('continueWatching', {'path': ['video_list', 'continueWatching'], 'loco_contexts': ['continueWatching'], - 'loco_known': True}), + 'loco_known': True, + 'label_id': 30168, + 'description_id': 30093, + 'icon': 'DefaultInProgressShows.png'}), ('newAndPopular', {'path': ['category_list', 'newAndPopular'], 'loco_contexts': ['comingSoon'], 'loco_known': False, @@ -85,7 +91,10 @@ class GlobalVariables: 'icon': 'DefaultRecentlyAddedMovies.png'}), ('chosenForYou', {'path': ['video_list', 'chosenForYou'], 'loco_contexts': ['topTen'], - 'loco_known': True}), + 'loco_known': True, + 'label_id': 30169, + 'description_id': 30094, + 'icon': 'DefaultUser.png'}), ('recentlyAdded', {'path': ['video_list_sorted', 'recentlyAdded', '1592210'], 'loco_contexts': None, 'loco_known': False, @@ -102,10 +111,16 @@ class GlobalVariables: 'query_without_reference': True}), ('currentTitles', {'path': ['video_list', 'currentTitles'], 'loco_contexts': ['trendingNow'], - 'loco_known': True}), + 'loco_known': True, + 'label_id': 30150, + 'description_id': 30146, + 'icon': 'DefaultRecentlyAddedMovies.png'}), ('mostViewed', {'path': ['video_list', 'mostViewed'], 'loco_contexts': ['popularTitles'], - 'loco_known': True}), + 'loco_known': True, + 'label_id': 30001, + 'description_id': 30094, + 'icon': 'DefaultUser.png'}), ('netflixOriginals', {'path': ['video_list_sorted', 'netflixOriginals', '839338'], 'loco_contexts': ['netflixOriginals'], 'loco_known': True, @@ -120,13 +135,6 @@ class GlobalVariables: 'icon': 'DefaultTVShows.png', 'has_sort_setting': True, 'query_without_reference': True}), - ('recommendations', {'path': ['recommendations', 'recommendations'], - 'loco_contexts': ['similars', 'becauseYouAdded', 'becauseYouLiked', 'watchAgain', - 'bigRow'], - 'loco_known': False, - 'label_id': 30001, - 'description_id': 30094, - 'icon': 'DefaultUser.png'}), ('tvshowsGenres', {'path': ['subgenres', 'tvshowsGenres', '83'], 'loco_contexts': None, 'loco_known': False, diff --git a/resources/lib/kodi/infolabels.py b/resources/lib/kodi/infolabels.py index f6ec0c387..ea34f89a6 100644 --- a/resources/lib/kodi/infolabels.py +++ b/resources/lib/kodi/infolabels.py @@ -55,8 +55,19 @@ def get_info(videoid, item, raw_data, profile_language_code='', delayed_db_op=Fa cache_entry = G.CACHE.get(CACHE_INFOLABELS, cache_identifier) infos = cache_entry['infos'] quality_infos = cache_entry['quality_infos'] + updated = False + updated = _refresh_missing_plot(infos, item) or updated + updated = _refresh_missing_atomic_infos(infos, item, ('Trailer', 'Year')) or updated + if videoid.mediatype == common.VideoId.EPISODE: + updated = _refresh_episode_numbers(infos, item) or updated + updated = _refresh_missing_referenced_infos(infos, item, raw_data) or updated + updated = _refresh_missing_profile_cast(infos, videoid, profile_language_code) or updated + if updated: + G.CACHE.add(CACHE_INFOLABELS, cache_identifier, {'infos': infos, 'quality_infos': quality_infos}, + delayed_db_op=delayed_db_op) except CacheMiss: infos, quality_infos = parse_info(videoid, item, raw_data, common_data) + _refresh_missing_profile_cast(infos, videoid, profile_language_code) G.CACHE.add(CACHE_INFOLABELS, cache_identifier, {'infos': infos, 'quality_infos': quality_infos}, delayed_db_op=delayed_db_op) # Use a deepcopy of dict to not reflect changes of the dictionary also to the cache @@ -68,10 +79,88 @@ def get_info(videoid, item, raw_data, profile_language_code='', delayed_db_op=Fa return infos_copy, quality_infos + +def _refresh_missing_plot(infos, item): + if infos.get('Plot') or infos.get('PlotOutline'): + return False + synopsis = common.get_path_safe(['synopsis', 'value'], item) + if not synopsis: + synopsis = common.get_path_safe(['regularSynopsis', 'value'], item) + if not synopsis: + return False + infos['Plot'] = synopsis + infos['PlotOutline'] = synopsis + return True + + +def _refresh_missing_referenced_infos(infos, item, raw_data): + if not item or not raw_data: + return False + updated = False + referenced_infos = _parse_referenced_infos(item, raw_data) + for key, value in referenced_infos.items(): + if value and not infos.get(key): + infos[key] = value + updated = True + return updated + + +def _refresh_missing_atomic_infos(infos, item, targets): + if not item: + return False + updated = False + atomic_infos = _parse_atomic_infos(item) + for key in targets: + value = atomic_infos.get(key) + if value and not infos.get(key): + infos[key] = value + updated = True + return updated + + +def _refresh_episode_numbers(infos, item): + """Refresh episode number infolabels from a newer episode item.""" + if not item: + return False + summary = item.get('summary', {}).get('value', {}) + if not isinstance(summary, dict): + return False + updated = False + for key, source in (('Season', 'season'), ('Episode', 'episode')): + value = summary.get(source) + if value is None: + continue + value = _transform_value(key, value) + if infos.get(key) != value: + infos[key] = value + updated = True + return updated + + +def _refresh_missing_profile_cast(infos, videoid, profile_language_code): + """Reuse language-independent cast names from another infolabel cache key.""" + if infos.get('Cast'): + return False + active_language = G.LOCAL_DB.get_profile_config('language', '') + for language_code in (active_language, ''): + if language_code == profile_language_code: + continue + try: + cached_infos = G.CACHE.get( + CACHE_INFOLABELS, f'{videoid.value}_{language_code}')['infos'] + except (CacheMiss, KeyError, TypeError): + continue + if cached_infos.get('Cast'): + infos['Cast'] = copy.deepcopy(cached_infos['Cast']) + return True + return False + + def add_info_list_item(list_item: ListItemW, videoid, item, raw_data, is_in_mylist, common_data, art_item=None, is_in_remind_me=False): """Add infolabels and art to a ListItem""" infos, quality_infos = get_info(videoid, item, raw_data, delayed_db_op=True, common_data=common_data) + _add_trailer_fallback(infos, videoid) list_item.addStreamInfoFromDict(quality_infos) if is_in_mylist and common_data.get('mylist_titles_color'): # Highlight ListItem title when the videoid is contained in "My list" @@ -87,6 +176,12 @@ def add_info_list_item(list_item: ListItemW, videoid, item, raw_data, is_in_myli delayed_db_op=True)) +def _add_trailer_fallback(infos, videoid): + if infos.get('Trailer') or videoid.mediatype not in (common.VideoId.MOVIE, common.VideoId.SHOW): + return + infos['Trailer'] = common.build_url(['play_trailer'], videoid=videoid, mode=G.MODE_ACTION) + + def _add_supplemental_plot_info(infos, item, common_data): """Add supplemental info to plot description""" suppl_info = [] @@ -132,6 +227,10 @@ def get_art(videoid, item, profile_language_code='', delayed_db_op=False): cache_identifier = f'{videoid.value}_{profile_language_code}' try: art = G.CACHE.get(CACHE_ARTINFO, cache_identifier) + parsed_art = parse_art(videoid, item) + if _refresh_cached_art(art, parsed_art, item): + G.CACHE.add(CACHE_ARTINFO, cache_identifier, art, + delayed_db_op=delayed_db_op) except CacheMiss: art = parse_art(videoid, item) G.CACHE.add(CACHE_ARTINFO, cache_identifier, art, @@ -139,6 +238,61 @@ def get_art(videoid, item, profile_language_code='', delayed_db_op=False): return art +def _refresh_cached_art(art, parsed_art, item): + updated = False + for key in ('poster', 'fanart', 'thumb', 'landscape', 'clearlogo'): + if parsed_art.get(key) and not art.get(key): + art[key] = parsed_art[key] + updated = True + if _repair_browser_boxart_wide_cache(art, parsed_art, item): + updated = True + return updated + + +def _repair_browser_boxart_wide_cache(art, parsed_art, item): + fallback = common.get_path_safe(['itemSummary', 'value', 'boxArt', 'url'], item) + updated = False + if fallback: + portrait_poster = parsed_art.get('poster') + if portrait_poster and portrait_poster != fallback: + for key in ('poster', 'thumb'): + if parsed_art.get(key) == portrait_poster and art.get(key) != portrait_poster: + art[key] = portrait_poster + updated = True + for key in ('fanart', 'thumb', 'landscape'): + if art.get(key) == fallback and parsed_art.get(key) != fallback: + art[key] = parsed_art.get(key, '') + updated = True + if art.get('poster') == fallback and parsed_art.get('poster') != fallback: + art['poster'] = parsed_art.get('poster', '') + updated = True + parsed_poster = parsed_art.get('poster') + if parsed_poster and parsed_poster != fallback and art.get('poster') != parsed_poster: + if not art.get('poster') or art.get('poster') in (art.get('landscape'), art.get('fanart'), fallback): + art['poster'] = parsed_poster + updated = True + elif art.get('thumb') == parsed_poster: + art['poster'] = parsed_poster + updated = True + elif parsed_poster and art.get('poster') and art['poster'] != parsed_poster: + if art['poster'] in (art.get('thumb'), art.get('landscape'), art.get('fanart')): + old_poster = art['poster'] + art['poster'] = parsed_poster + if art.get('thumb') == old_poster: + art['thumb'] = parsed_poster + updated = True + parsed_thumb = parsed_art.get('thumb') + if (parsed_thumb and parsed_thumb == parsed_poster and parsed_thumb != fallback + and art.get('thumb') != parsed_thumb): + art['thumb'] = parsed_thumb + updated = True + elif parsed_thumb and art.get('thumb') and art['thumb'] != parsed_thumb: + if art['thumb'] in (art.get('landscape'), art.get('fanart'), fallback): + art['thumb'] = parsed_thumb + updated = True + return updated + + def get_resume_info_from_library(videoid): """Retrieve the resume value from the Kodi library""" try: @@ -265,15 +419,20 @@ def parse_art(videoid, item): def _assign_art(videoid, **kwargs): """Assign the art available from Netflix to appropriate Kodi art""" - art = {'poster': _best_art([kwargs['poster'], kwargs['fallback']]), - 'fanart': _best_art([kwargs['fanart'], - kwargs['interesting_moment'], - kwargs['boxart_large'], - kwargs['boxart_small']]), - 'thumb': ((kwargs['interesting_moment'] - if videoid.mediatype in (common.VideoId.EPISODE, common.VideoId.SUPPLEMENTAL) else '') - or kwargs['boxart_large'] or kwargs['boxart_small'])} - art['landscape'] = art['thumb'] + poster = _best_art([kwargs['poster'], kwargs['fallback']]) + wide_art = _best_art([kwargs['interesting_moment'], kwargs['boxart_large'], kwargs['boxart_small']]) + if videoid.mediatype in (common.VideoId.EPISODE, common.VideoId.SUPPLEMENTAL): + thumb = wide_art + elif videoid.mediatype == common.VideoId.UNSPECIFIED: + # Video-list/category folders may only have a landscape preview from the + # browser-shaped response. Kodi skins commonly render these folders via thumb. + thumb = poster or wide_art + else: + thumb = poster + art = {'poster': poster, + 'fanart': _best_art([kwargs['fanart'], wide_art]), + 'thumb': thumb} + art['landscape'] = wide_art or thumb if videoid.mediatype != common.VideoId.UNSPECIFIED: art['clearlogo'] = _best_art([kwargs['clearlogo']]) return art @@ -317,32 +476,42 @@ def set_watched_status(list_item: ListItemW, video_data, common_data): resume_time = 0 video_runtime = video_data.get('runtime', {}).get('value', 0) if is_watched_user_overrided is None: - # Note to shakti properties: - # 'watched': unlike the name this value is used to other purposes, so not to set a video as watched - # 'watchedToEndOffset': this value is used to determine if a video is watched but - # is available only with the metadata api and only for "episode" video type - # 'creditsOffset' : this value is used as position where to show the (play) "Next" (episode) button - # on the website, but it may not be always available with the "movie" video type - credits_offset_val = video_data.get('creditsOffset', {}).get('value', 0) - if credits_offset_val > 0: - # To better ensure that a video is marked as watched also when a user do not reach the ending credits - # we generally lower the watched threshold by 50 seconds for 50 minutes of video (3000 secs) - lower_value = video_runtime / 3000 * 50 - watched_threshold = credits_offset_val - lower_value - else: - # When missing the value should be only a video of movie type, - # then we simulate the default Kodi playcount behaviour (playcountminimumpercent) - watched_threshold = video_runtime / 100 * 90 - # To avoid asking to the server again the entire list of titles (after watched a video) - # to get the updated value, we override the value with the value saved in memory (see am_video_events.py) + # Cached bookmarks are written by AMVideoEvents while playback is active. They must + # override list data, including GraphQL fallback items, which remains stale until the + # server response and directory caches are refreshed. + has_cached_bookmark = True try: bookmark_position = G.CACHE.get(CACHE_BOOKMARKS, video_id) except CacheMiss: + has_cached_bookmark = False # NOTE shakti 'bookmarkPosition' tag when it is not set have -1 value - bookmark_position = video_data['bookmarkPosition'].get('value', 0) - playcount = 1 if 0 < watched_threshold <= bookmark_position else 0 - if playcount == 0 and bookmark_position > 0: - resume_time = bookmark_position + bookmark_position = video_data.get('bookmarkPosition', {}).get('value', 0) + graphql_playcount = video_data.get('_graphql_playcount', {}).get('value') + if graphql_playcount is not None and not has_cached_bookmark: + playcount = int(graphql_playcount) + if playcount == 0 and bookmark_position > 0: + resume_time = bookmark_position + else: + # Note to shakti properties: + # 'watched': unlike the name this value is used to other purposes, so not to set a video as watched + # 'watchedToEndOffset': this value is used to determine if a video is watched but + # is available only with the metadata api and only for "episode" video type + # 'creditsOffset' : this value is used as position where to show the (play) "Next" (episode) button + # on the website, but it may not be always available with the "movie" video type + credits_offset_val = (video_data.get('creditsOffset', {}).get('value', 0) or + video_data.get('watchedToEndOffset', {}).get('value', 0)) + if credits_offset_val > 0: + # To better ensure that a video is marked as watched also when a user do not reach the ending credits + # we generally lower the watched threshold by 50 seconds for 50 minutes of video (3000 secs) + lower_value = video_runtime / 3000 * 50 + watched_threshold = credits_offset_val - lower_value + else: + # When missing the value should be only a video of movie type, + # then we simulate the default Kodi playcount behaviour (playcountminimumpercent) + watched_threshold = video_runtime / 100 * 90 + playcount = 1 if 0 < watched_threshold <= bookmark_position else 0 + if playcount == 0 and bookmark_position > 0: + resume_time = bookmark_position else: playcount = 1 if is_watched_user_overrided else 0 # We have to set playcount with setInfo(), because the setProperty('PlayCount', ) have a bug diff --git a/resources/lib/navigation/actions.py b/resources/lib/navigation/actions.py index 75ecf5a78..72c577316 100644 --- a/resources/lib/navigation/actions.py +++ b/resources/lib/navigation/actions.py @@ -9,6 +9,8 @@ """ import xbmc import xbmcgui +import xbmcplugin +from urllib.parse import unquote, urlparse import resources.lib.common as common import resources.lib.kodi.ui as ui @@ -114,9 +116,10 @@ def my_list(self, videoid, pathitems): operation = pathitems[1] api.update_my_list(videoid, operation, self.params) sync_library(videoid, operation) - if operation == 'remove' and common.WndHomeProps[common.WndHomeProps.CURRENT_DIRECTORY_MENU_ID] == 'myList': + is_mylist = common.WndHomeProps[common.WndHomeProps.CURRENT_DIRECTORY_MENU_ID] == 'myList' + if operation == 'remove' and is_mylist: common.json_rpc('Input.Down') # Avoids selection back to the top - common.container_refresh() + common.container_refresh() @common.inject_video_id(path_offset=1) def remind_me(self, videoid): @@ -142,14 +145,75 @@ def trailer(self, videoid): 'video_id_dict': video_id_dict, 'supplemental_type': SUPPLEMENTAL_TYPE_TRAILERS }) - if list_data: + if list_data or self._direct_trailer_url(videoid): url = common.build_url(['supplemental'], params={'video_id_dict': dumps(video_id_dict), 'supplemental_type': SUPPLEMENTAL_TYPE_TRAILERS}, mode=G.MODE_DIRECTORY) common.container_update(url) - else: - ui.show_notification(common.get_local_string(30111)) + return + ui.show_notification(common.get_local_string(30111)) + + @common.inject_video_id(path_offset=1) + @measure_exec_time_decorator() + def play_trailer(self, videoid): + """Resolve the first trailer for the Kodi info dialog trailer button.""" + trailer_videoid = self._first_trailer_videoid(videoid) + if trailer_videoid: + from resources.lib.navigation.player import _play # pylint: disable=protected-access,import-outside-toplevel + _play(trailer_videoid, False) + return + self._resolve_direct_trailer(videoid) + + @common.inject_video_id(path_offset=1) + @measure_exec_time_decorator() + def play_direct_trailer(self, videoid): + """Resolve only the direct metadata trailer selected from the trailer menu.""" + self._resolve_direct_trailer(videoid) + + def _resolve_direct_trailer(self, videoid): + trailer_url = self._direct_trailer_url(videoid) + if trailer_url: + title = xbmc.getInfoLabel('ListItem.Title') or xbmc.getInfoLabel('ListItem.Label') + list_item = xbmcgui.ListItem(title, path=trailer_url, offscreen=True) + if title: + list_item.setInfo('video', {'Title': title}) + list_item.setProperty('isPlayable', 'true') + list_item.setContentLookup(False) + xbmcplugin.setResolvedUrl(handle=G.PLUGIN_HANDLE, succeeded=True, listitem=list_item) + return True + xbmcplugin.setResolvedUrl(handle=G.PLUGIN_HANDLE, succeeded=False, listitem=xbmcgui.ListItem()) + return False + + @staticmethod + def _first_trailer_videoid(videoid): + menu_data = {'path': ['is_context_menu_item', 'is_context_menu_item'], + 'title': common.get_local_string(30179)} + list_data, _extra_data = common.make_call('get_video_list_supplemental', + { + 'menu_data': menu_data, + 'video_id_dict': videoid.to_dict(), + 'supplemental_type': SUPPLEMENTAL_TYPE_TRAILERS + }) + for url, list_item, is_folder in list_data or []: + if is_folder: + continue + videoid_path = list_item.getProperty('nf_videoid') + if videoid_path: + return common.VideoId.from_path(videoid_path.split('/')) + parsed_path = _plugin_pathitems(url) + if parsed_path[:1] == [G.MODE_PLAY]: + return common.VideoId.from_path(parsed_path[1:]) + return None + + @staticmethod + def _direct_trailer_url(videoid): + try: + direct_trailer = common.make_call('get_direct_trailer', videoid) or {} + except Exception as exc: # pylint: disable=broad-except + LOG.warn('Trailer info lookup failed for {}: {}', videoid, exc) + return '' + return direct_trailer.get('url', '') @measure_exec_time_decorator() def purge_cache(self, pathitems=None): # pylint: disable=unused-argument @@ -293,3 +357,8 @@ def change_watched_status_locally(videoid): G.SHARED_DB.set_watched_status(profile_guid, videoid.value, True) ui.show_notification(common.get_local_string(30237).split('|')[txt_index]) common.container_refresh() + + +def _plugin_pathitems(url): + path = unquote(urlparse(url).path).strip('/') + return [part for part in path.split('/') if part] diff --git a/resources/lib/navigation/directory.py b/resources/lib/navigation/directory.py index 95a3ec741..f0586430e 100644 --- a/resources/lib/navigation/directory.py +++ b/resources/lib/navigation/directory.py @@ -7,6 +7,7 @@ SPDX-License-Identifier: MIT See LICENSES/MIT.md for more information. """ +import xbmcgui import xbmcplugin import resources.lib.common as common @@ -156,10 +157,12 @@ def video_list(self, pathitems): menu_data = G.MAIN_MENU_ITEMS.get(pathitems[1]) if not menu_data: # Dynamic menus menu_data = G.LOCAL_DB.get_value(pathitems[1], table=TABLE_MENU_DATA, data_type=dict) + list_id = pathitems[2] if len(pathitems) > 2 else pathitems[1] + is_dynamic_id = len(pathitems) > 2 and not G.is_known_menu_context(list_id) call_args = { - 'list_id': pathitems[2], + 'list_id': list_id, 'menu_data': menu_data, - 'is_dynamic_id': not G.is_known_menu_context(pathitems[2]) + 'is_dynamic_id': is_dynamic_id } dir_items, extra_data = common.make_call('get_video_list', call_args) @@ -180,7 +183,7 @@ def video_list_sorted(self, pathitems): 'menu_data': menu_data, 'sub_genre_id': self.params.get('sub_genre_id'), # Used to show the sub-genre folder when sub-genres exists 'perpetual_range_start': self.perpetual_range_start, - 'is_dynamic_id': not G.is_known_menu_context(pathitems[2]) + 'is_dynamic_id': len(pathitems) > 2 and not G.is_known_menu_context(pathitems[2]) } dir_items, extra_data = common.make_call('get_video_list_sorted', call_args) sort_type = 'sort_nothing' @@ -239,6 +242,28 @@ def supplemental(self, pathitems): # pylint: disable=unused-argument 'supplemental_type': self.params['supplemental_type'] } dir_items, extra_data = common.make_call('get_video_list_supplemental', call_args) + if not dir_items: + videoid = common.VideoId.from_dict(call_args['video_id_dict']) + direct_trailer = common.make_call('get_direct_trailer', videoid) or {} + trailer_url = direct_trailer.get('url', '') + if trailer_url: + title = direct_trailer.get('title') or common.get_local_string(30179) + list_item = xbmcgui.ListItem(title, path=trailer_url, offscreen=True) + infos = {'Title': title} + if direct_trailer.get('synopsis'): + infos['Plot'] = direct_trailer['synopsis'] + infos['PlotOutline'] = direct_trailer['synopsis'] + if direct_trailer.get('year'): + infos['Year'] = direct_trailer['year'] + list_item.setInfo('video', infos) + if direct_trailer.get('poster'): + list_item.setArt({ + 'poster': direct_trailer['poster'], + 'thumb': direct_trailer['poster'] + }) + list_item.setProperty('isPlayable', 'true') + list_item.setContentLookup(False) + dir_items = [(trailer_url, list_item, False)] finalize_directory(dir_items, menu_data.get('content_type', G.CONTENT_SHOW), title=get_title(menu_data, extra_data)) diff --git a/resources/lib/navigation/directory_search.py b/resources/lib/navigation/directory_search.py index f9b243e56..16e570ccf 100644 --- a/resources/lib/navigation/directory_search.py +++ b/resources/lib/navigation/directory_search.py @@ -101,18 +101,10 @@ def search_add(): row_id = _search_add_bygenreid(SEARCH_TYPES[type_index], genre_id) else: raise NotImplementedError(f'Search type index {type_index} not implemented') - # Redirect to "search" endpoint (otherwise no results in JSON-RPC) - # Rewrite path history using dir_update_listing + container_update - # (otherwise will retrigger input dialog on Back or Container.Refresh) - if row_id is not None and search_query(row_id, 0, False): - url = common.build_url(['search', 'search', row_id], mode=G.MODE_DIRECTORY, params={'dir_update_listing': True}) - from time import sleep - # The forced sleep its needed because seem that change the container path too fast - # make problems in Kodi core and the GUI fails to update, when this happens cause side effects to context menus - # like "add/remove from my list" that when used ask again to make a new search because re-open the initial path - sleep(1) - common.container_update(url, False) - return True + # Replace the current listing synchronously. An asynchronous Container.Update + # races the still-open /add directory and Kodi can discard the results. + if row_id is not None: + return search_query(str(row_id), 0, True) return False diff --git a/resources/lib/services/nfsession/directorybuilder/dir_builder.py b/resources/lib/services/nfsession/directorybuilder/dir_builder.py index 67e9c8165..ae8b3b4c8 100644 --- a/resources/lib/services/nfsession/directorybuilder/dir_builder.py +++ b/resources/lib/services/nfsession/directorybuilder/dir_builder.py @@ -7,15 +7,26 @@ SPDX-License-Identifier: MIT See LICENSES/MIT.md for more information. """ -from resources.lib.utils.data_types import merge_data_type -from resources.lib.common.exceptions import CacheMiss +from concurrent.futures import ThreadPoolExecutor, as_completed + +import resources.lib.common as common +from resources.lib.utils.data_types import merge_data_type, CustomVideoList +from resources.lib.common.cache_utils import CACHE_COMMON +from resources.lib.common.exceptions import CacheMiss, InvalidVideoListTypeError from resources.lib.common import VideoId from resources.lib.globals import G +from resources.lib.utils.api_paths import ART_SIZE_FHD, ART_SIZE_POSTER from resources.lib.services.nfsession.directorybuilder.dir_builder_items \ import (build_video_listing, build_subgenres_listing, build_season_listing, build_episode_listing, build_loco_listing, build_mainmenu_listing, build_profiles_listing, build_lolomo_category_listing) -from resources.lib.services.nfsession.directorybuilder.dir_path_requests import DirectoryPathRequests -from resources.lib.utils.logging import measure_exec_time_decorator +from resources.lib.services.nfsession.directorybuilder.dir_path_requests import (DirectoryPathRequests, + _has_reference_entries, + _metadata_has_reference_names, + _metadata_image_url, + _metadata_year, + metadata_with_title_page_fallback, + normalize_metadata_references) +from resources.lib.utils.logging import LOG, measure_exec_time_decorator class DirectoryBuilder(DirectoryPathRequests): @@ -76,32 +87,226 @@ def get_episodes(self, pathitems, seasonid_dict, perpetual_range_start): @measure_exec_time_decorator(is_immediate=True) def get_video_list(self, list_id, menu_data, is_dynamic_id): - if not is_dynamic_id: - list_id = self.get_loco_list_id_by_context(menu_data['loco_contexts'][0]) - # pylint: disable=unexpected-keyword-arg - video_list = self.req_video_list(list_id, no_use_cache=menu_data.get('no_use_cache')) + menu_id = menu_data['path'][1] + defer_title_details = ( + is_dynamic_id + and menu_data.get('initial_menu_id') == 'newAndPopular') + cache_enriched_list = ( + (defer_title_details or (not is_dynamic_id and menu_id == 'chosenForYou')) + and not menu_data.get('no_use_cache')) + enriched_cache_id = f'enriched_video_list_{list_id}' + video_list = None + if cache_enriched_list: + try: + video_list = G.CACHE.get(CACHE_COMMON, enriched_cache_id) + except CacheMiss: + pass + current_contexts = { + 'currentTitles': ('windowedNewReleases',), + 'mostViewed': ('mostWatched',) + } + if video_list is None: + if not is_dynamic_id and menu_id == 'continueWatching': + video_list = self._browser_continue_watching_list() + elif not is_dynamic_id and menu_id == 'chosenForYou': + video_list = self._browser_top_picks_list() + elif not is_dynamic_id and menu_id in current_contexts: + video_list = self._video_list_from_lolomo_category_context( + 'comingSoon', current_contexts[menu_id], fallback_first=True) + else: + if not is_dynamic_id: + list_id = self.get_loco_list_id_by_context(menu_data['loco_contexts'][0]) + # pylint: disable=unexpected-keyword-arg + video_list = self.req_video_list( + list_id, menu_data=menu_data, no_use_cache=menu_data.get('no_use_cache')) + if menu_id != 'continueWatching': + # New & Popular rows can contain dozens of titles. The browser + # response already supplies each title and contextual artwork, + # so block only on repairing genuinely missing posters instead + # of risking the 20-second directory timeout. + self._enrich_video_list_art( + video_list, + include_refs=not defer_title_details, + art_only=defer_title_details) + if cache_enriched_list: + G.CACHE.add(CACHE_COMMON, enriched_cache_id, video_list) return build_video_listing(video_list, menu_data, mylist_items=self.req_mylist_items()) @measure_exec_time_decorator(is_immediate=True) def get_video_list_sorted(self, pathitems, menu_data, sub_genre_id, perpetual_range_start, is_dynamic_id): context_id = None - if is_dynamic_id and pathitems[2] != 'None': + if is_dynamic_id and len(pathitems) > 2 and pathitems[2] != 'None': # Dynamic IDs for common video lists # The context_id can be: # -In the loco list: 'video list id' # -In the video list: 'sub-genre id' # -In the list of genres: 'sub-genre id' context_id = pathitems[2] - # pylint: disable=unexpected-keyword-arg - video_list = self.req_video_list_sorted(menu_data['request_context_name'], - context_id=context_id, - perpetual_range_start=perpetual_range_start, - menu_data=menu_data, - no_use_cache=menu_data.get('no_use_cache')) + if menu_data['path'][1] == 'recentlyAdded' and context_id: + video_list = self._video_list_from_lolomo_category_context( + 'comingSoon', ('windowedNewReleases', 'newThisWeek', 'newOnNetflix', 'newOnNetflixThisWeek'), + fallback_first=True) + self._filter_unavailable_videos(video_list) + else: + # pylint: disable=unexpected-keyword-arg + video_list = self.req_video_list_sorted(menu_data['request_context_name'], + context_id=context_id, + perpetual_range_start=perpetual_range_start, + menu_data=menu_data, + no_use_cache=menu_data.get('no_use_cache')) + self._enrich_video_list_art(video_list, include_refs=True) return build_video_listing(video_list, menu_data, sub_genre_id, pathitems, perpetual_range_start, self.req_mylist_items()) + def _enrich_video_list_art(self, video_list, include_refs=False, art_only=False): + if not getattr(video_list, 'videos', None): + return video_list + pending = [] + for video in video_list.videos.values(): + if not isinstance(video, dict): + continue + needs_art = self._needs_metadata_boxart(video) + needs_refs = include_refs and not _has_reference_entries(video, 'cast') + needs_year = (not art_only and + not common.get_path_safe(['releaseYear', 'value'], video)) + needs_synopsis = (not art_only and + not (common.get_path_safe(['synopsis', 'value'], video) or + common.get_path_safe(['regularSynopsis', 'value'], video))) + if not needs_art and not needs_refs and not needs_year and not needs_synopsis: + continue + try: + videoid = VideoId.from_videolist_item(video) + except Exception: # pylint: disable=broad-except + continue + if videoid.mediatype not in (VideoId.MOVIE, VideoId.SHOW): + continue + pending.append((videoid, video, needs_art, needs_refs, needs_year, needs_synopsis)) + if not pending: + return video_list + + try: + metadata_request = self._prepare_metadata_request() + except Exception as exc: # pylint: disable=broad-except + LOG.debug('List metadata request setup failed ({})', type(exc).__name__) + metadata_request = None + + def _load_metadata(item): + videoid, _video, _needs_art, needs_refs, needs_year, needs_synopsis = item + try: + metadata = (self._metadata_for_video_from_request(videoid.value, metadata_request) + if metadata_request else {}) + except Exception as exc: # pylint: disable=broad-except + LOG.debug('Metadata enrichment skipped for {}: {}', videoid, exc) + metadata = {} + if (not art_only and + ((needs_refs and not _metadata_has_reference_names(metadata)) or + (needs_year and not _metadata_year(metadata)) or + (needs_synopsis and not self._metadata_synopsis(metadata)))): + metadata = metadata_with_title_page_fallback(videoid.value, metadata) + return item, metadata + + max_workers = min(6, len(pending)) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(_load_metadata, item) for item in pending] + for future in as_completed(futures): + try: + item, metadata = future.result() + except Exception as exc: # pylint: disable=broad-except + LOG.debug('Metadata enrichment worker failed ({})', type(exc).__name__) + continue + videoid, video, needs_art, needs_refs, needs_year, needs_synopsis = item + if needs_art: + self._apply_metadata_art(video, metadata) + if needs_synopsis: + self._apply_metadata_synopsis(video, metadata) + if needs_year: + release_year = _metadata_year(metadata) + if release_year: + video['releaseYear'] = {'value': release_year} + if needs_refs: + normalize_metadata_references(video_list.data, videoid.value, metadata, video) + video_list.artitem = next(iter(video_list.videos.values()), None) + return video_list + + @staticmethod + def _needs_metadata_boxart(video): + poster = common.get_path_safe(['boxarts', ART_SIZE_POSTER, 'jpg', 'value', 'url'], video) + return not poster + + @staticmethod + def _apply_metadata_art(video, metadata): + boxart = DirectoryBuilder._best_metadata_art( + metadata, ('boxart', 'boxArt', 'boxarts'), portrait=True) + if boxart: + video.setdefault('boxarts', {})[ART_SIZE_POSTER] = {'jpg': {'value': {'url': boxart}}} + wide_art = DirectoryBuilder._best_metadata_art( + metadata, ('artwork', 'interestingMoment', 'storyart', 'storyArt'), portrait=False) + if wide_art: + video.setdefault('interestingMoment', {})[ART_SIZE_FHD] = {'jpg': {'value': {'url': wide_art}}} + + @staticmethod + def _metadata_synopsis(metadata): + if not isinstance(metadata, dict): + return '' + return metadata.get('synopsis') or metadata.get('regularSynopsis') or '' + + @staticmethod + def _apply_metadata_synopsis(video, metadata): + synopsis = DirectoryBuilder._metadata_synopsis(metadata) + if synopsis: + video['synopsis'] = {'value': synopsis} + video['regularSynopsis'] = {'value': synopsis} + + @staticmethod + def _best_metadata_art(metadata, keys, portrait): + return _metadata_image_url(metadata, keys, portrait) + + def _filter_unavailable_videos(self, video_list): + videos_type = type(video_list.videos) + playable_videos = videos_type( + (video_id, video) + for video_id, video in video_list.videos.items() + if video.get('availability', {}).get('value', {}).get('isPlayable', False)) + if len(playable_videos) == len(video_list.videos): + return video_list + video_list.videos = playable_videos + video_list.artitem = next(iter(playable_videos.values()), None) + video_list.contained_titles = [ + video.get('title', {}).get('value') + for video in playable_videos.values() + if video.get('title', {}).get('value')] + return video_list + + def _video_list_from_lolomo_category_context(self, category_name, contexts, fallback_first=False): + if isinstance(contexts, str): + contexts = (contexts,) + first_list_id = None + for list_id, summary, video_list in self.req_lolomo_category(category_name).lists(): + if not first_list_id and video_list.videos: + first_list_id = list_id + if summary.get('context') in contexts: + return self._browser_lolomo_video_list_by_id(category_name, list_id) + if fallback_first and first_list_id: + return self._browser_lolomo_video_list_by_id(category_name, first_list_id) + raise InvalidVideoListTypeError(f'No LoLoMo category list with context {contexts} available') + + def _video_list_from_genre_context(self, genre_id, contexts): + if isinstance(contexts, str): + contexts = (contexts,) + try: + loco_list = self.req_loco_list_genre(genre_id) + for list_id, video_list in loco_list.lists.items(): + if video_list.get('context') in contexts: + try: + return self._browser_genre_video_list_by_id(genre_id, list_id) + except Exception as exc: # pylint: disable=broad-except + LOG.warn('Using materialized genre row {} after list lookup failed: {}', list_id, exc) + return video_list + except Exception as exc: + LOG.warn('Continue Watching genre fallback failed: {}', exc) + return CustomVideoList({'videos': {}}) + @measure_exec_time_decorator(is_immediate=True) def get_video_list_sorted_sp(self, pathitems, menu_data, context_name, context_id, perpetual_range_start): # Method used for the menu search @@ -131,14 +336,21 @@ def get_video_list_chunked(self, pathitems, menu_data, chunked_video_list, perpe @measure_exec_time_decorator(is_immediate=True) def get_video_list_search(self, pathitems, menu_data, search_term, perpetual_range_start, path_params=None): video_list = self.req_video_list_search(search_term, perpetual_range_start=perpetual_range_start) + # Search already uses browser GraphQL result art. Extra metadata/My List lookups can exceed the IPC timeout. return build_video_listing(video_list, menu_data, - pathitems=pathitems, mylist_items=self.req_mylist_items(), path_params=path_params) + pathitems=pathitems, mylist_items=[], path_params=path_params) @measure_exec_time_decorator(is_immediate=True) def get_genres(self, menu_data, genre_id, force_use_videolist_id): if genre_id: # Load the LoCo list of the specified genre loco_list = self.req_loco_list_genre(genre_id) + if menu_data['path'][1] in ('tvshows', 'movies'): + menu_data = dict(menu_data) + menu_data['loco_contexts'] = None + force_use_videolist_id = True + elif menu_data['path'][1] == 'recommendations': + return build_lolomo_category_listing(self.req_lolomo_category('comingSoon'), menu_data) else: # Load the LoCo root list filtered by 'loco_contexts' specified in the menu_data loco_list = self.req_loco_list_root() @@ -166,7 +378,11 @@ def add_videoids_to_video_list_cache(self, cache_bucket, cache_identifier, video """Add the specified video ids to a video list datatype in the cache (only if the cache item exists)""" try: video_list_sorted_data = G.CACHE.get(cache_bucket, cache_identifier) - merge_data_type(video_list_sorted_data, self.req_datatype_video_list_byid(video_ids)) + data_to_merge = self.req_datatype_video_list_byid(video_ids) + for video in data_to_merge.videos.values(): + video.setdefault('queue', {'value': {}}) + video['queue'].setdefault('value', {})['inQueue'] = True + merge_data_type(video_list_sorted_data, data_to_merge) G.CACHE.add(cache_bucket, cache_identifier, video_list_sorted_data) except CacheMiss: pass @@ -178,6 +394,11 @@ def get_continuewatching_videoid_exists(self, video_id): :param video_id: videoid as [string] value :return: a tuple ([bool] true if videoid exists, [string] the current list id, that depends from loco id) """ - list_id = self.get_loco_list_id_by_context('continueWatching') - video_list = self.req_video_list(list_id).videos if video_id else [] + try: + list_id = self.get_loco_list_id_by_context('continueWatching') + video_list = self.req_video_list(list_id).videos if video_id else [] + except Exception: + current_list = self._video_list_from_genre_context('1592210', ('continueWatching',)) + list_id = current_list.videoid.value if getattr(current_list, 'videoid', None) else None + video_list = current_list.videos if video_id else [] return video_id in video_list, list_id diff --git a/resources/lib/services/nfsession/directorybuilder/dir_builder_items.py b/resources/lib/services/nfsession/directorybuilder/dir_builder_items.py index 0ef5c52c6..a3e2e0aee 100644 --- a/resources/lib/services/nfsession/directorybuilder/dir_builder_items.py +++ b/resources/lib/services/nfsession/directorybuilder/dir_builder_items.py @@ -50,12 +50,23 @@ def build_mainmenu_listing(loco_list): continue if data['loco_known']: list_id, video_list = loco_list.find_by_context(data['loco_contexts'][0]) - if not list_id: + if list_id: + menu_title = video_list['displayName'] + directory_item = _create_videolist_item(list_id, video_list, data, common_data, static_lists=True) + directory_item[1].addContextMenuItems(generate_context_menu_mainmenu(menu_id)) + directory_items.append(directory_item) + elif data.get('label_id'): + menu_title = common.get_local_string(data['label_id']) + menu_description = (common.get_local_string(data['description_id']) + if data.get('description_id') is not None + else '') + list_item = ListItemW(label=menu_title) + list_item.setArt({'icon': data.get('icon', 'DefaultFolder.png')}) + list_item.setInfo('video', {'Plot': menu_description}) + list_item.addContextMenuItems(generate_context_menu_mainmenu(menu_id)) + directory_items.append((common.build_url(data['path'], mode=G.MODE_DIRECTORY), list_item, True)) + else: continue - menu_title = video_list['displayName'] - directory_item = _create_videolist_item(list_id, video_list, data, common_data, static_lists=True) - directory_item[1].addContextMenuItems(generate_context_menu_mainmenu(menu_id)) - directory_items.append(directory_item) else: menu_title = common.get_local_string(data['label_id']) if data.get('label_id') else 'Missing menu title' menu_description = (common.get_local_string(data['description_id']) @@ -213,6 +224,11 @@ def build_loco_listing(loco_list, menu_data, force_use_videolist_id=False): sub_menu_data['force_use_videolist_id'] = force_use_videolist_id sub_menu_data['title'] = video_list['displayName'] sub_menu_data['initial_menu_id'] = menu_data.get('initial_menu_id', menu_data['path'][1]) + if menu_data.get('path', [None])[0] == 'genres' and len(menu_data['path']) > 2: + sub_menu_data['browser_genre_id'] = ( + str(video_list['genreId']) + if video_list['context'] == 'genre' + else str(menu_data['path'][2])) # Do not use the cache with 'Top 10' menus, so that you always get up-to-date data. sub_menu_data['no_use_cache'] = video_list['context'] == 'mostWatched' G.LOCAL_DB.set_value(list_id, sub_menu_data, TABLE_MENU_DATA) @@ -262,7 +278,8 @@ def build_video_listing(video_list, menu_data, sub_genre_id=None, pathitems=None 'active_profile_guid': G.LOCAL_DB.get_active_profile_guid(), 'marks_tvshow_started': G.ADDON.getSettingBool('marks_tvshow_started'), 'trackid': trackid, - 'is_supplemental_type': video_list.__class__.__name__ == 'VideoListSupplemental' + 'is_supplemental_type': (getattr(video_list, 'is_supplemental_type', False) or + video_list.__class__.__name__ == 'VideoListSupplemental') }) directory_items = [_create_video_item(videoid_value, video, video_list, perpetual_range_start, common_data) for videoid_value, video diff --git a/resources/lib/services/nfsession/directorybuilder/dir_path_requests.py b/resources/lib/services/nfsession/directorybuilder/dir_path_requests.py index ee6c2bee9..081d2dd77 100644 --- a/resources/lib/services/nfsession/directorybuilder/dir_path_requests.py +++ b/resources/lib/services/nfsession/directorybuilder/dir_path_requests.py @@ -8,20 +8,956 @@ See LICENSES/MIT.md for more information. """ from typing import TYPE_CHECKING +from collections import OrderedDict +from concurrent.futures import ThreadPoolExecutor, as_completed +from html.parser import HTMLParser +from types import SimpleNamespace +import json +import re +import time +import uuid +from urllib.parse import urlencode, urljoin + +import requests +import requests.exceptions as req_exceptions from resources.lib import common +from resources.lib.utils import website from resources.lib.utils.data_types import (VideoListSorted, SubgenreList, SeasonList, EpisodeList, LoCo, VideoList, - SearchVideoList, CustomVideoList, LoLoMoCategory, VideoListSupplemental, + CustomVideoList, LoLoMoCategory, VideoListSupplemental, VideosList) -from resources.lib.common.exceptions import InvalidVideoListTypeError, InvalidVideoId +from resources.lib.common.exceptions import (InvalidVideoListTypeError, InvalidVideoId, MetadataNotAvailable, + WebsiteParsingError) +from resources.lib.database.db_utils import TABLE_SESSION from resources.lib.utils.api_paths import (VIDEO_LIST_PARTIAL_PATHS, RANGE_PLACEHOLDER, VIDEO_LIST_BASIC_PARTIAL_PATHS, SEASONS_PARTIAL_PATHS, EPISODES_PARTIAL_PATHS, ART_PARTIAL_PATHS, - TRAILER_PARTIAL_PATHS, PATH_REQUEST_SIZE_STD, build_paths, - PATH_REQUEST_SIZE_MAX, jgraph_get) + ART_SIZE_FHD, ART_SIZE_POSTER, TRAILER_PARTIAL_PATHS, + SUPPLEMENTAL_TYPE_TRAILERS, build_paths, PATH_REQUEST_SIZE_MAX) from resources.lib.common import cache_utils from resources.lib.globals import G +from resources.lib.services.nfsession.session.endpoints import ENDPOINTS from resources.lib.utils.logging import LOG +GRAPHQL_URL = 'https://web.prod.cloud.netflix.com/graphql' +GRAPHQL_OP_SEASONS = 'dbc3b274-d4f9-4811-aaf1-d082d3b936f2' +GRAPHQL_OP_EPISODES = '27b30e4e-871d-46aa-ac8b-244103d2e37d' +GRAPHQL_OP_SEARCH = '8d902979-56f2-4886-8c16-f8910f6b52ee' +GRAPHQL_OP_CAROUSEL_PAGE = 'cbe70fd8-c3a1-4e1b-9ab3-d690850ad7f3' +GRAPHQL_OP_DETAIL_MODAL = '7265187b-c065-4714-9eee-327751c8e215' +GRAPHQL_OP_DETAIL_MODAL_TRAILERS = '06e30ee5-7983-4fef-8135-5914124b76ad' +TOP_PICKS_SECTION_LABEL = 'top picks' +NETFLIX_TITLE_URL = 'https://www.netflix.com/title/{}' +TITLE_PAGE_GRAPHQL_RE = re.compile(r"netflix\.reactContext\.models\.graphql\s*=\s*JSON\.parse\('(.*?)'\);", re.DOTALL) +TITLE_PAGE_JSONLD_RE = re.compile( + r']+type=["\']application/ld\+json["\'][^>]*>(.*?)', re.DOTALL) +LOCO_ROOT_ID_RE = re.compile(r'NES_[A-Za-z0-9_]+_p_\d+') +LOCO_ROOT_CANDIDATE_RE = re.compile(r'NES_[A-Za-z0-9_]+') +LOCO_ROW_RANGE = {'from': 0, 'to': 50} +LOCO_PAGE_RANGE = {'from': 0, 'to': 20} +LOCO_REFERENCE_FIELDS = [ + 'availability', 'episodeCount', 'inRemindMeList', 'queue', 'summary', + 'title', 'synopsis', 'runtime', 'seasonCount', 'bookmarkPosition', + 'creditsOffset', 'watched', 'delivery', 'trackIds', 'userRating', + 'maturity', 'releaseYear' +] +LOCO_CATEGORY_CONTEXTS = { + 'comingSoon': ('newThisWeek', 'popularTitles', 'mostWatched', 'trendingNow'), + 'recommendations': ('similars', 'becauseYouAdded', 'becauseYouLiked', 'watchAgain', 'bigRow', + 'topTen', 'trendingNow', 'popularTitles') +} +SORTED_LIST_CONTEXT_FALLBACKS = { + ('genres', '1592210'): 'newThisWeek' +} +BROWSER_LOCO_ROW_KEYS = [0, 1, 2, 3, 'continueWatching'] +BROWSER_LOCO_OTHER_ROW_KEYS = [1, 2, 3, 'continueWatching'] +BROWSER_LOCO_SUMMARY_FIELDS = [ + 'availability', 'bbSupplementalMessage', 'bbSupplementalMessageIcon', + 'maturity', 'mostWatchedData', 'summary' +] +BROWSER_LOCO_CURRENT_FIELDS = ['hasAudioDescription', 'summary'] +BROWSER_LOCO_CONTINUE_FIELDS = ['bookmarkPosition', 'runtime', 'summary', 'title'] +BROWSER_LOCO_REFERENCE_FIELDS = ['availability', 'episodeCount', 'inRemindMeList', 'queue', 'summary'] +BROWSER_LOCO_METADATA_FIELDS = BROWSER_LOCO_REFERENCE_FIELDS + [ + 'title', 'synopsis', 'runtime', 'seasonCount', 'bookmarkPosition', + 'creditsOffset', 'watched', 'delivery', 'trackIds', 'userRating', + 'maturity', 'releaseYear', 'promoVideo' +] +BROWSER_LOCO_PERSON_FIELDS = ['genres', 'tags', 'creators', 'directors', 'cast'] +BROWSER_GENRE_SUBGENRE_FIELDS = ['id', 'name', 'unifiedEntityId'] +BROWSER_LOCO_DIRECT_RANGE = {'from': 0, 'to': PATH_REQUEST_SIZE_MAX} +BROWSER_LOCO_HOME_ROW_RANGE = {'from': 4, 'to': 50} +BROWSER_LOCO_HOME_VISIBLE_RANGE = {'from': 0, 'to': 8} +BROWSER_LOCO_CONTINUE_LAZY_RANGE = {'from': 8, 'to': 100} +BROWSER_MYLIST_RANGE = {'from': 0, 'to': 48} +BROWSER_MYLIST_FIELDS = [ + 'availability', 'episodeCount', 'inRemindMeList', 'itemSummary', + 'queue', 'summary' +] +SEARCH_GRAPHQL_PAGE_SIZE = 48 +SEARCH_TITLE_PAGE_METADATA_LIMIT = SEARCH_GRAPHQL_PAGE_SIZE +SEARCH_TITLE_PAGE_METADATA_WORKERS = 12 +METADATA_REFERENCE_KEYS = { + 'cast': ('people', ('cast', 'actors', 'actor', 'starring', 'starringActors')), + 'directors': ('people', ('directors', 'director')), + 'creators': ('people', ('creators', 'creator', 'writers', 'writer')), + 'genres': ('genres', ('genres', 'genre', 'tags')) +} + + +class _ActiveProfileLinkParser(HTMLParser): + """Find the profile switch link without exposing its token in logs.""" + + def __init__(self, active_profile_guid): + super().__init__(convert_charrefs=True) + self.active_profile_guid = str(active_profile_guid or '').lower() + self.href = None + + def handle_starttag(self, tag, attrs): + if self.href or tag != 'a' or not self.active_profile_guid: + return + attributes = dict(attrs) + href = attributes.get('href') or '' + classes = (attributes.get('class') or '').split() + if self.active_profile_guid in href.lower() and ( + 'profile-link' in classes or '/switchprofile' in href.lower()): + self.href = href + + +def _value(value): + return {'value': value} + + +def _has_reference_entries(item, source): + refs = item.get(source, {}) if isinstance(item, dict) else {} + if not isinstance(refs, dict): + return False + return any(common.is_numeric(key) for key in refs) + + +def _metadata_names_from_value(value): + if not value: + return [] + if isinstance(value, str): + return [name.strip() for name in value.split(',') if name.strip()] + if isinstance(value, list): + names = [] + for item in value: + names.extend(_metadata_names_from_value(item)) + return names + if not isinstance(value, dict): + return [] + for key in ('name', 'fullName', 'displayName', 'title'): + name = value.get(key) + if isinstance(name, str) and name.strip(): + return [name.strip()] + if isinstance(name, dict): + nested_names = _metadata_names_from_value(name.get('value') or name) + if nested_names: + return nested_names + for key in ('value', 'person', 'node'): + nested_names = _metadata_names_from_value(value.get(key)) + if nested_names: + return nested_names + if 'edges' in value: + return _metadata_names_from_value(value.get('edges')) + if all(common.is_numeric(key) for key in value): + names = [] + for item in value.values(): + names.extend(_metadata_names_from_value(item)) + return names + return [] + + +def _metadata_names(metadata, keys): + names = [] + for key in keys: + names.extend(_metadata_names_from_value(metadata.get(key))) + unique_names = [] + seen = set() + for name in names: + normalized = name.strip() + if not normalized or normalized.lower() in seen: + continue + seen.add(normalized.lower()) + unique_names.append(normalized) + return unique_names + + +def _metadata_has_reference_names(metadata): + if not isinstance(metadata, dict): + return False + for _source, (_target, keys) in METADATA_REFERENCE_KEYS.items(): + if _metadata_names(metadata, keys): + return True + return False + + +def _metadata_has_trailer(metadata): + return bool(_metadata_trailer_id(metadata) or _metadata_trailer_url(metadata)) + + +def _metadata_year(metadata): + if not isinstance(metadata, dict): + return 0 + value = (metadata.get('year') or metadata.get('releaseYear') or + metadata.get('dateCreated') or metadata.get('datePublished')) + if isinstance(value, int): + return value + match = re.search(r'\b(18|19|20|21)\d{2}\b', str(value or '')) + return int(match.group(0)) if match else 0 + + +def _title_page_jsonld_data(content): + from html import unescape + html_text = content.decode('utf-8', 'replace') if isinstance(content, bytes) else str(content) + for match in TITLE_PAGE_JSONLD_RE.finditer(html_text): + try: + jsonld_data = json.loads(unescape(match.group(1))) + except (TypeError, ValueError) as exc: + LOG.debug('Unable to parse title page JSON-LD ({})', type(exc).__name__) + continue + candidates = jsonld_data if isinstance(jsonld_data, list) else [jsonld_data] + for candidate in candidates: + if not isinstance(candidate, dict): + continue + if candidate.get('@type') in ('Movie', 'TVSeries') or candidate.get('actors') or candidate.get('creators'): + return candidate + return {} + + +def _metadata_from_title_page(video_id): + try: + response = requests.get( + NETFLIX_TITLE_URL.format(video_id), + headers={ + 'Accept': 'text/html,application/xhtml+xml,application/xml', + 'User-Agent': common.get_user_agent(enable_android_mediaflag_fix=True) + }, + timeout=8) + response.raise_for_status() + except req_exceptions.RequestException as exc: + LOG.debug('Title page metadata fallback failed for {} ({})', video_id, type(exc).__name__) + return {} + return _title_page_jsonld_data(response.content) + + +def metadata_with_title_page_fallback(video_id, metadata_video=None): + """Return metadata enriched with public title page JSON-LD fields.""" + metadata_video = dict(metadata_video or {}) + if (_metadata_has_reference_names(metadata_video) and + _metadata_has_trailer(metadata_video) and + _metadata_year(metadata_video) and + (metadata_video.get('synopsis') or metadata_video.get('regularSynopsis'))): + return metadata_video + title_page_metadata = _metadata_from_title_page(video_id) + return _merge_title_page_metadata(metadata_video, title_page_metadata) + + +def _merge_title_page_metadata(metadata_video, title_page_metadata): + metadata_video = dict(metadata_video or {}) + if not title_page_metadata: + return metadata_video + for key in ('actors', 'directors', 'creators', 'genre', 'trailer'): + if key in title_page_metadata and key not in metadata_video: + metadata_video[key] = title_page_metadata[key] + description = title_page_metadata.get('description') + if description and not (metadata_video.get('synopsis') or metadata_video.get('regularSynopsis')): + metadata_video['synopsis'] = description + metadata_video['regularSynopsis'] = description + if not _metadata_year(metadata_video): + title_page_year = _metadata_year(title_page_metadata) + if title_page_year: + metadata_video['year'] = title_page_year + return metadata_video + + +def _search_title_page_metadata(video_id): + try: + response = requests.get( + NETFLIX_TITLE_URL.format(video_id), + headers={ + 'Accept': 'text/html,application/xhtml+xml,application/xml', + 'User-Agent': common.get_user_agent(enable_android_mediaflag_fix=True) + }, + timeout=(2, 4)) + response.raise_for_status() + except req_exceptions.RequestException as exc: + LOG.debug('Search title page metadata skipped for {} ({})', video_id, type(exc).__name__) + return {} + return _title_page_jsonld_data(response.content) + + +def _metadata_trailer_id(metadata_video): + promo_video = metadata_video.get('promoVideo') + if isinstance(promo_video, dict): + promo_value = promo_video.get('value') if isinstance(promo_video.get('value'), dict) else promo_video + trailer_id = promo_value.get('id') or promo_value.get('videoId') + if trailer_id: + return trailer_id + trailer_id = metadata_video.get('merchedVideoId') or metadata_video.get('promoVideoId') + return trailer_id + + +def _metadata_trailer_url(metadata_video): + trailer = metadata_video.get('trailer') + if isinstance(trailer, dict): + trailer_url = trailer.get('contentUrl') or trailer.get('url') + if trailer_url: + return trailer_url + return metadata_video.get('trailerUrl') or metadata_video.get('previewUrl') + + +def _add_metadata_trailer(item, metadata_video): + if item.get('promoVideo') or item.get('trailerUrl'): + return + trailer_id = _metadata_trailer_id(metadata_video) + if trailer_id: + item['promoVideo'] = _value({'id': trailer_id}) + return + trailer_url = _metadata_trailer_url(metadata_video) + if trailer_url: + item['trailerUrl'] = _value(trailer_url) + + +def _reference_id(prefix, name, index): + safe_name = re.sub(r'[^A-Za-z0-9]+', '_', name).strip('_').lower() + return f'{prefix}_{index}_{safe_name}' if safe_name else f'{prefix}_{index}' + + +def _add_metadata_references(path_response, item, source, target, names): + if not names or _has_reference_entries(item, source): + return + target_data = path_response.setdefault(target, {}) + refs = item.setdefault(source, {}) + for index, name in enumerate(names[:10]): + ref_id = _reference_id(f'metadata_{source}', name, index) + target_data.setdefault(ref_id, {'name': _value(name)}) + refs[str(index)] = {'$type': 'ref', 'value': [target, ref_id]} + + +def normalize_metadata_references(path_response, video_id, metadata_video, item=None): + """Copy metadata people/genre fields into the JSON graph reference shape.""" + if not isinstance(metadata_video, dict): + return + item = item or path_response.get('videos', {}).get(str(video_id)) + if not isinstance(item, dict): + return + for source, (target, keys) in METADATA_REFERENCE_KEYS.items(): + _add_metadata_references(path_response, item, source, target, _metadata_names(metadata_video, keys)) + _add_metadata_trailer(item, metadata_video) + + +def _summary(video_id, title, video_type, number=None, length=None): + data = {'id': int(video_id), 'type': video_type, 'name': title} + if number is not None: + data['season' if video_type == 'season' else 'episode'] = number + data['shortName'] = str(number) + if length is not None: + data['length'] = length + return _value(data) + + +def _graphql_headers(): + headers = { + 'Accept': '*/*', + 'Content-Type': 'application/json', + 'Origin': 'https://www.netflix.com', + 'Referer': 'https://www.netflix.com/browse', + 'x-netflix.nq.stack': 'prod', + 'x-netflix.request.client.user.guid': G.LOCAL_DB.get_active_profile_guid() + } + for header, key in ( + ('X-Netflix.browserVersion', 'browser_info_version'), + ('X-Netflix.osName', 'browser_info_os_name'), + ('X-Netflix.osVersion', 'browser_info_os_version'), + ('X-Netflix.uiVersion', 'ui_version')): + value = G.LOCAL_DB.get_value(key, '', table=('session', ['Name', 'Value'])) + if value: + headers[header] = value + return headers + + +def _season_node_to_item(node, index): + season_id = str(node['videoId']) + title = node.get('title') or f'Season {index + 1}' + episodes = node.get('episodes', {}).get('totalCount') + return season_id, { + 'summary': _summary(season_id, title, 'season', index + 1, episodes), + 'title': _value(title), + 'availability': _value({'isPlayable': True}) + } + + +def _episode_node_to_item(node, season_number, metadata=None): + episode_id = str(node['videoId']) + metadata = metadata or {} + synopsis = metadata.get('synopsis') or (node.get('contextualSynopsis') or {}).get('text') or '' + runtime = metadata.get('runtime') or node.get('runtimeSec') or node.get('displayRuntimeSec') or 0 + bookmark = metadata.get('bookmark') or node.get('bookmark') or {} + bookmark_position = bookmark.get('offset') or bookmark.get('bookmarkPosition') or 0 + credits_offset = metadata.get('creditsOffset') or metadata.get('watchedToEndOffset') or 0 + watched_threshold = credits_offset - (runtime / 3000 * 50) if credits_offset else runtime * 0.9 + graph_playcount = 1 if 0 < watched_threshold <= bookmark_position else 0 + artwork = node.get('artwork') or {} + image_url = '' + if isinstance(artwork, dict): + image_url = artwork.get('url') or (artwork.get('image') or {}).get('url') or '' + summary = _summary(episode_id, node.get('title') or '', 'episode', node.get('number')) + if season_number is not None: + summary['value']['season'] = season_number + return episode_id, { + 'summary': summary, + 'title': _value(node.get('title') or ''), + 'synopsis': _value(synopsis), + 'regularSynopsis': _value(synopsis), + 'runtime': _value(runtime), + 'availability': _value({'isPlayable': bool(node.get('isPlayable', True))}), + 'bookmarkPosition': _value(bookmark_position), + 'creditsOffset': _value(metadata.get('creditsOffset', 0)), + 'watchedToEndOffset': _value(metadata.get('watchedToEndOffset', 0)), + 'watched': _value(bool(bookmark.get('watchedDate'))), + '_graphql_playcount': _value(graph_playcount), + 'interestingMoment': {'_1920x1080': {'jpg': {'value': {'url': image_url}}}}, + 'season': _value(season_number) + } + + +def _search_graphql_artwork_params(): + return { + 'artworkType': 'BOXSHOT', + 'dimension': {'width': 300, 'height': 420}, + 'features': {'fallbackStrategy': 'STILL'} + } + + +def _search_graphql_game_artwork_params(artwork_type, top_content_type_badge): + return { + 'artworkType': artwork_type, + 'dimension': {'width': 342, 'height': 192}, + 'features': {'fallbackStrategy': 'STILL', 'topContentTypeBadge': top_content_type_badge} + } + + +def _search_graphql_options(): + entity_treatments = { + 'pinotStandardBoxshot': {'base': {'canHandleEntityKinds': ['VIDEO']}}, + 'pinotStandardCloudAppIcon': {'base': {'canHandleEntityKinds': ['GAME']}}, + 'pinotStandardMobileAppIcon': {'base': {'canHandleEntityKinds': ['GAME']}}, + 'pinotStandardDestination': {'base': {'canHandleEntityKinds': ['GENERIC_CONTAINER']}} + } + return { + 'pageCapabilities': {'base': { + 'canHandlePlayingCloudGames': False, + 'capabilitiesBySection': { + 'pinotGallery': {'base': {'capabilitiesBySectionTreatment': { + 'pinotCreatorHome': {'base': { + 'capabilitiesByEntityTreatment': entity_treatments, + 'maxTotalEntities': 300 + }}, + 'pinotStandard': {'base': { + 'capabilitiesByEntityTreatment': entity_treatments, + 'maxTotalEntities': 300 + }} + }}}, + 'pinotList': {'base': {'capabilitiesBySectionTreatment': { + 'pinotSuggestions': {'base': { + 'capabilitiesByEntityTreatment': { + 'pinotSuggestion': {'base': {'canHandleEntityKinds': [ + 'AUTOCOMPLETE', 'VIDEO', 'CHARACTER', 'GENERIC_CONTAINER', 'GENRE', 'PERSON' + ]}} + }, + 'maxTotalEntities': 100 + }} + }}} + }, + 'maxTotalSections': 2 + }}, + 'session': {'id': str(uuid.uuid4())} + } + + +def _search_graphql_variables(search_term, end_cursor=None): + return { + 'imageParamsForStandardBoxart': _search_graphql_artwork_params(), + 'imageParamsForCloudGameBoxart': _search_graphql_game_artwork_params( + 'GAME_CLOUD_BOXART_HORIZONTAL_INCOMPATIBLE', True), + 'imageParamsForMobileGameBoxart': _search_graphql_game_artwork_params( + 'GAME_ICON_BOXART_HORIZONTAL_CARD', True), + 'pageSize': SEARCH_GRAPHQL_PAGE_SIZE, + 'options': _search_graphql_options(), + 'searchTerm': search_term, + 'selectedSuggestionId': None, + 'endCursor': end_cursor + } + + +def _merge_search_metadata_video(base_video, metadata_video): + merged = dict(base_video) + metadata_video = metadata_video or {} + title = metadata_video.get('title') or merged.get('title', {}).get('value') + if title: + merged['title'] = _value(title) + summary = merged.get('summary', {}).get('value', {}) + if isinstance(summary, dict): + summary['name'] = title + merged['summary'] = _value(summary) + synopsis = metadata_video.get('synopsis') or metadata_video.get('regularSynopsis') + if synopsis: + merged['synopsis'] = _value(synopsis) + merged['regularSynopsis'] = _value(synopsis) + runtime = metadata_video.get('runtime') + if runtime: + merged['runtime'] = _value(runtime) + release_year = _metadata_year(metadata_video) + if release_year: + merged['releaseYear'] = _value(release_year) + seasons = metadata_video.get('seasons') or [] + if seasons: + merged['seasonCount'] = _value(len(seasons)) + episode_count = sum(len(season.get('episodes') or []) for season in seasons) + if episode_count: + merged['episodeCount'] = _value(episode_count) + poster_url = _metadata_image_url(metadata_video, ('boxart', 'boxArt'), portrait=True) + if poster_url: + boxarts = dict(merged.get('boxarts') or {}) + boxarts[ART_SIZE_POSTER] = { + 'jpg': {'value': {'url': poster_url}} + } + merged['boxarts'] = boxarts + landscape_url = _metadata_image_url( + metadata_video, ('artwork', 'storyart', 'storyArt', 'stills'), portrait=False) + if landscape_url: + interesting_moments = dict(merged.get('interestingMoment') or {}) + interesting_moments[ART_SIZE_FHD] = { + 'jpg': {'value': {'url': landscape_url}} + } + merged['interestingMoment'] = interesting_moments + return merged + + +def _metadata_video_to_item(video_id, metadata_video): + title = metadata_video.get('title') or str(video_id) + video_type = str(metadata_video.get('type') or metadata_video.get('videoType') or '').lower() + if video_type not in ('movie', 'show'): + video_type = 'show' if metadata_video.get('seasons') else 'movie' + base_video = { + 'summary': _summary(str(video_id), title, video_type), + 'title': _value(title), + 'availability': _value({'isPlayable': True}), + 'queue': _value({'inQueue': False}), + 'inRemindMeList': _value(False), + 'bookmarkPosition': _value(0), + 'creditsOffset': _value(0), + 'watchedToEndOffset': _value(0), + 'watched': _value(False), + 'runtime': _value(0), + 'releaseYear': _value(0), + 'maturity': _value({}), + 'trackIds': _value({}), + 'requestId': _value('') + } + return _merge_search_metadata_video(base_video, metadata_video) + + +def _metadata_image_url(metadata, keys, portrait): + candidates = [] + + def _collect(value): + if isinstance(value, str): + if value.startswith('http'): + candidates.append((value, 0, 0)) + return + if isinstance(value, list): + for item in value: + _collect(item) + return + if not isinstance(value, dict): + return + url = value.get('url') + if isinstance(url, str) and url.startswith('http'): + width = value.get('w') or value.get('width') or 0 + height = value.get('h') or value.get('height') or 0 + candidates.append((url, width, height)) + else: + for nested_value in value.values(): + _collect(nested_value) + + for key in keys: + _collect(metadata.get(key) if isinstance(metadata, dict) else None) + if not candidates: + return '' + matching = [ + candidate for candidate in candidates + if candidate[1] and candidate[2] and + ((candidate[2] > candidate[1]) if portrait else (candidate[1] > candidate[2])) + ] + if matching: + return max(matching, key=lambda candidate: candidate[1] * candidate[2])[0] + unknown_size = [candidate for candidate in candidates if not candidate[1] or not candidate[2]] + return unknown_size[0][0] if unknown_size else '' + + +def _search_graphql_node_to_item(node): + entity = node.get('unifiedEntity') or {} + entity_type = entity.get('__typename') + if entity_type not in ('Movie', 'Show'): + return None + video_id = str(entity.get('videoId') or '') + if not video_id: + return None + video_type = 'movie' if entity_type == 'Movie' else 'show' + title = node.get('displayString') or str(video_id) + item = { + 'summary': _summary(video_id, title, video_type), + 'title': _value(title), + 'availability': _value({'isPlayable': True}), + 'queue': _value({'inQueue': False}), + 'inRemindMeList': _value(False), + 'bookmarkPosition': _value(0), + 'creditsOffset': _value(0), + 'watchedToEndOffset': _value(0), + 'watched': _value(False), + 'runtime': _value(entity.get('runtimeSec', 0)), + 'releaseYear': _value(entity.get('releaseYear', 0)), + 'maturity': _value(entity.get('contentAdvisory') or {}), + 'trackIds': _value({}), + 'requestId': _value('') + } + artwork = (node.get('contextualArtwork') or {}).get('artwork') or {} + if artwork.get('url'): + _set_browser_boxart(item, { + 'id': int(video_id), + 'title': title, + 'boxArt': { + 'url': artwork['url'], + 'width': artwork.get('width') or artwork.get('w'), + 'height': artwork.get('height') or artwork.get('h') + } + }) + return video_id, item + + +def _carousel_graphql_variables(row_id, end_cursor): + return { + 'rowId': row_id, + 'carouselAfterCursor': end_cursor, + 'carouselPageSize': 12, + 'eddEnabled': False, + 'imageParamsForStandardBoxart': { + 'artworkType': 'SDP', + 'dimension': {'width': 342, 'height': 192}, + 'features': {'fallbackStrategy': 'STILL'} + }, + 'imageParamsForRankedBoxart': { + 'artworkType': 'BOXSHOT', + 'dimension': {'width': 426, 'height': 607}, + 'features': {'fallbackStrategy': 'STILL', 'suppressTop10Badge': True} + }, + 'imageParamsForContinueWatchingBoxart': { + 'artworkType': 'SDP', + 'dimension': {'width': 342, 'height': 192}, + 'features': {'fallbackStrategy': 'STILL'} + }, + 'imageParamsForMobileGameBoxart': { + 'artworkType': 'APP_ICON', + 'dimension': {'width': 200, 'height': 200}, + 'formats': ['WEBP', 'JPG', 'PNG'] + }, + 'imageParamsForCloudGameBoxart': { + 'artworkType': 'SDP', + 'dimension': {'width': 342, 'height': 192}, + 'features': {'fallbackStrategy': 'STILL'} + }, + 'imageParamsForCharacterCircle': { + 'artworkType': 'SQUAREHEADSHOT_1000x1000', + 'dimension': {'width': 200, 'height': 200}, + 'formats': ['WEBP', 'JPG', 'PNG'] + }, + 'carouselVersion': '1' + } + + + +def _graphql_cache_node(graphql_data, typename, video_id): + video_id = str(video_id) + direct_key = f'{typename}:{{"videoId":{video_id}}}' + node = graphql_data.get(direct_key) + if isinstance(node, dict): + return node + key_prefix = f'{typename}:' + for key, candidate in graphql_data.items(): + if not isinstance(candidate, dict): + continue + if key.startswith(key_prefix) and str(candidate.get('videoId')) == video_id: + return candidate + return None + + +def _graphql_ref_node(graphql_data, node_or_ref): + if not isinstance(node_or_ref, dict): + return None + ref = node_or_ref.get('__ref') + if ref: + return graphql_data.get(ref) + return node_or_ref + + +def _iter_graphql_edges(value): + if isinstance(value, dict) and '__ref' in value: + return [] + edges = value.get('edges') if isinstance(value, dict) else value + if isinstance(edges, dict): + return edges.values() + if isinstance(edges, list): + return edges + return [] + + +def _first_artwork_url(value): + if isinstance(value, dict): + url = value.get('url') + if isinstance(url, str) and url.startswith('http'): + return url + for nested in value.values(): + nested_url = _first_artwork_url(nested) + if nested_url: + return nested_url + elif isinstance(value, list): + for nested in value: + nested_url = _first_artwork_url(nested) + if nested_url: + return nested_url + return '' + + +def _supplemental_artwork_url(node): + for key, value in node.items(): + if not any(name in key.lower() for name in ('artwork', 'boxart', 'storyart', 'still')): + continue + image_url = _first_artwork_url(value) + if image_url: + return image_url + return '' + + +def _supplemental_node_to_item(node): + video_id = str(node.get('videoId') or node.get('id') or '') + # DetailModalTrailers is authoritative for collection playability. Do not + # turn an explicitly unplayable (or incomplete) card into a playable Kodi + # item merely because it has a video id. + if not video_id or node.get('isPlayable') is not True: + return None + title = node.get('title') or node.get('displayName') or video_id + item = { + 'summary': _summary(video_id, title, 'movie'), + 'title': _value(title), + 'availability': _value({'isPlayable': True}), + 'queue': _value({'inQueue': False}), + 'inRemindMeList': _value(False), + 'bookmarkPosition': _value(0), + 'creditsOffset': _value(0), + 'watchedToEndOffset': _value(0), + 'watched': _value(False), + 'runtime': _value(node.get('displayRuntimeSec') or node.get('runtimeSec') or node.get('runtime') or 0), + 'trackIds': _value({'trackId': video_id}), + 'requestId': _value('') + } + synopsis = node.get('synopsis') or node.get('contextualSynopsis') or '' + if isinstance(synopsis, dict): + synopsis = synopsis.get('text') or synopsis.get('value') or '' + if synopsis: + item['synopsis'] = _value(synopsis) + item['regularSynopsis'] = _value(synopsis) + image_url = _supplemental_artwork_url(node) + if image_url: + _set_browser_boxart(item, {'id': int(video_id), 'title': title, 'boxArt': {'url': image_url}}) + return video_id, item + + +def _supplemental_videos_from_graphql_cache(graphql_data, video_id): + if not isinstance(graphql_data, dict): + return OrderedDict() + title_node = (_graphql_cache_node(graphql_data, 'Show', video_id) or + _graphql_cache_node(graphql_data, 'Movie', video_id)) + if not isinstance(title_node, dict): + return OrderedDict() + supplemental_list = title_node.get('supplementalVideosList') or {} + supplemental_list = _graphql_ref_node(graphql_data, supplemental_list) or supplemental_list + videos = OrderedDict() + for edge in _iter_graphql_edges(supplemental_list): + edge_node = edge.get('node') if isinstance(edge, dict) else edge + supplemental_node = _graphql_ref_node(graphql_data, edge_node) + if not isinstance(supplemental_node, dict): + continue + item = _supplemental_node_to_item(supplemental_node) + if item: + videos[item[0]] = item[1] + return videos + + +def _title_page_graphql_data(content, react_context): + graphql_data = common.get_path_safe(['models', 'graphql', 'data'], react_context, False, {}) + if isinstance(graphql_data, dict) and graphql_data: + return graphql_data + html = content.decode('utf-8', 'replace') if isinstance(content, bytes) else str(content) + match = TITLE_PAGE_GRAPHQL_RE.search(html) + if not match: + return {} + try: + graphql_cache = json.loads(website.decode_javascript_string(match.group(1))) + except (TypeError, ValueError, UnicodeDecodeError) as exc: + LOG.warn('Unable to parse title page GraphQL cache ({})', type(exc).__name__) + return {} + graphql_data = graphql_cache.get('data') if isinstance(graphql_cache, dict) else None + return graphql_data if isinstance(graphql_data, dict) else {} + + +def _normalize_browser_list_lengths(path_response): + for list_data in path_response.get('lists', {}).values(): + if not isinstance(list_data, dict): + continue + length = list_data.get('componentSummary', {}).get('value', {}).get('length') + if not isinstance(length, int): + continue + for key in list(list_data.keys()): + if common.is_numeric(key) and int(key) >= length: + del list_data[key] + + +def _browser_reference_paths(reference_path, include_metadata=False): + fields = BROWSER_LOCO_METADATA_FIELDS if include_metadata else BROWSER_LOCO_REFERENCE_FIELDS + paths = [reference_path + [fields]] + if include_metadata: + paths.append(reference_path + [BROWSER_LOCO_PERSON_FIELDS, {'from': 0, 'to': 10}, ['id', 'name']]) + return paths + + +def _boxart_dimensions(boxart): + try: + width = int(boxart.get('width') or boxart.get('w') or 0) + height = int(boxart.get('height') or boxart.get('h') or 0) + except (TypeError, ValueError): + return 0, 0 + return width, height + + +def _set_browser_boxart(video, item_summary): + boxart = item_summary.get('boxArt') or {} + image_url = boxart.get('url') + width, height = _boxart_dimensions(boxart) + is_portrait = bool(width and height and height > width) + item_summary_value = dict(item_summary) + if image_url and not is_portrait: + item_summary_value.pop('boxArt', None) + video['itemSummary'] = _value(item_summary_value) + if not image_url: + return + art_value = {'url': image_url} + if is_portrait: + video.setdefault('boxarts', {}) + video['boxarts'].setdefault(ART_SIZE_POSTER, {'jpg': {'value': art_value}}) + else: + video.setdefault('interestingMoment', {}) + video['interestingMoment'].setdefault(ART_SIZE_FHD, {'jpg': {'value': art_value}}) + + +def _browser_item_summary_score(item_summary): + synopses = item_summary.get('synopses') or {} + synopsis = (synopses.get('regularSynopsis') or synopses.get('shortSynopsis') or + synopses.get('narrative')) + return bool(synopsis), len(synopsis or ''), len(item_summary) + + +def _normalize_browser_video_fields(path_response): + _normalize_browser_list_lengths(path_response) + item_summaries = {} + for list_data in path_response.get('lists', {}).values(): + if not isinstance(list_data, dict): + continue + for item in list_data.values(): + if not isinstance(item, dict): + continue + item_summary = item.get('itemSummary', {}).get('value', {}) + ref = item.get('reference', {}) + ref_value = ref.get('value') if isinstance(ref, dict) else ref + if isinstance(ref_value, dict) and 'value' in ref_value: + ref_value = ref_value['value'] + if isinstance(ref_value, list) and len(ref_value) >= 2 and ref_value[0] == 'videos': + video_id = str(ref_value[1]) + current_summary = item_summaries.get(video_id, {}) + if _browser_item_summary_score(item_summary) > _browser_item_summary_score(current_summary): + item_summaries[video_id] = item_summary + for video_id, video in path_response.get('videos', {}).items(): + if not isinstance(video, dict): + continue + item_summary = item_summaries.get(str(video_id), {}) + if not item_summary: + item_summary = video.get('itemSummary', {}).get('value', {}) + if item_summary: + video.setdefault('itemSummary', _value(item_summary)) + _set_browser_boxart(video, item_summary) + current = video.get('current', {}) + if isinstance(current, dict): + for key in BROWSER_LOCO_CONTINUE_FIELDS: + if key in current and key not in video: + video[key] = current[key] + summary = video.get('summary', {}).get('value', {}) + if not isinstance(summary, dict): + summary = {} + title_data = video.get('title') + title_value = title_data.get('value') if isinstance(title_data, dict) else title_data + if not title_value: + nested_title = summary.get('title') + if isinstance(nested_title, dict): + title_value = nested_title.get('value') + elif nested_title: + title_value = nested_title + else: + title_value = (item_summary.get('title') or item_summary.get('name') or + item_summary.get('displayName') or summary.get('name') or str(video_id)) + video['title'] = _value(title_value) + if title_value and isinstance(summary, dict): + summary.setdefault('name', title_value) + synopses = item_summary.get('synopses') or {} + synopsis = synopses.get('regularSynopsis') or synopses.get('shortSynopsis') or synopses.get('narrative') + if synopsis: + video.setdefault('synopsis', _value(synopsis)) + video.setdefault('regularSynopsis', _value(synopsis)) + video.setdefault('availability', _value(item_summary.get('availability', {'isPlayable': True}))) + video.setdefault('queue', _value({'inQueue': False})) + video.setdefault('inRemindMeList', _value(False)) + video.setdefault('bookmarkPosition', _value(0)) + video.setdefault('creditsOffset', _value(0)) + video.setdefault('watchedToEndOffset', _value(0)) + video.setdefault('watched', _value(False)) + video.setdefault('runtime', _value(summary.get('runtime', item_summary.get('runtime', item_summary.get('infoDensityRuntime', 0))))) + video.setdefault('releaseYear', _value(item_summary.get('releaseYear', 0))) + video.setdefault('seasonCount', _value(item_summary.get('seasonCount', 0))) + video.setdefault('episodeCount', _value(item_summary.get('episodeCount', 0))) + video.setdefault('maturity', _value(item_summary.get('maturity', {}))) + video.setdefault('trackIds', _value({})) + video.setdefault('requestId', _value(item_summary.get('requestId', ''))) + + +def _browser_list_video_ids(path_response, list_id): + video_ids = [] + list_data = path_response.get('lists', {}).get(str(list_id), {}) + for key, item in list_data.items(): + if not common.is_numeric(key) or not isinstance(item, dict): + continue + ref = item.get('reference', {}) + ref_value = ref.get('value') if isinstance(ref, dict) else ref + if isinstance(ref_value, dict) and 'value' in ref_value: + ref_value = ref_value['value'] + video_id = None + if isinstance(ref_value, list) and len(ref_value) >= 2 and ref_value[0] == 'videos': + video_id = ref_value[1] + if video_id is None: + video_id = item.get('itemSummary', {}).get('value', {}).get('videoId') + if video_id is not None and str(video_id) not in video_ids: + video_ids.append(str(video_id)) + return video_ids + if TYPE_CHECKING: # This variable/imports are used only by the editor, so not at runtime from resources.lib.services.nfsession.nfsession_ops import NFSessionOperations @@ -37,16 +973,17 @@ def req_mylist_items(self): """Return the 'my list' video list as videoid items""" LOG.debug('Requesting "my list" video list as videoid items') try: - items = [] - video_list = self.req_datatype_video_list_full(G.MAIN_MENU_ITEMS['myList']['request_context_name']) + video_list = self._browser_mylist_video_list() if video_list: - # pylint: disable=unused-variable - items = [common.VideoId.from_videolist_item(video) - for video_id, video in video_list.videos.items() - if video['queue']['value'].get('inQueue', False)] - return items + return [common.VideoId.from_videolist_item(video) + for video in video_list.videos.values()] except InvalidVideoListTypeError: return [] + except req_exceptions.HTTPError as exc: + if getattr(exc.response, 'status_code', None) != 404: + raise + LOG.warn('My List marker lookup disabled because pathEvaluator returned 404') + return [] @cache_utils.cache_output(cache_utils.CACHE_COMMON, fixed_identifier='loco_list', ignore_self_class=True) def req_loco_list_root(self): @@ -55,44 +992,28 @@ def req_loco_list_root(self): # - To get items for the main menu # (when 'loco_known'==True and loco_contexts is set, see MAIN_MENU_ITEMS in globals.py) # - To get list items for menus that have multiple contexts set to 'loco_contexts' like 'recommendations' menu - LOG.debug('Requesting LoCo\'s root ID') - paths = ([['loco', 'componentSummary']]) + LOG.debug('Requesting LoCo root lists') + paths = ([['loco', 'componentSummary'], + ['loco', {'from': 0, 'to': 50}, 'componentSummary'], + # Titles of first 4 videos in each video list (needed only to show titles in the plot description) + ['loco', {'from': 0, 'to': 50}, {'from': 0, 'to': 3}, 'reference', ['title', 'summary']]] + + # Art for the first video of each context list (needed only to add art to the menu item) + build_paths(['loco', {'from': 0, 'to': 50}, 0, 'reference'], ART_PARTIAL_PATHS)) call_args = {'paths': paths} - loco_response = self.nfsession.path_request(**call_args) - loco_data = jgraph_get('loco', loco_response) - comp_data = jgraph_get('componentSummary', loco_data) - loco_id = comp_data.get('id') - if loco_id: - LOG.debug('Requesting LoCo root lists') - paths = ([['locos', loco_id, {'from': 0, 'to': 50}, 'componentSummary'], - # Titles of first 4 videos in each video list (needed only to show titles in the plot description) - ['locos', loco_id, {'from': 0, 'to': 50}, {'from': 0, 'to': 3}, 'reference', ['title', 'summary']]]) - # Art for the first video of each context list (needed only to add art to the menu item) - # + build_paths(['locos', loco_id, {'from': 0, 'to': 50}, 0, 'reference'], ART_PARTIAL_PATHS)) - call_args = {'paths': paths} + try: path_response = self.nfsession.path_request(**call_args) - return LoCo(path_response) - else: - LOG.error('Cannot get the LoCo\'s root ID, response data: {}', loco_response) - return LoCo({}) + except req_exceptions.HTTPError as exc: + if exc.response is None or exc.response.status_code != 404: + raise + LOG.warn('Falling back to empty LoCo root menu after pathEvaluator 404') + path_response = {'locos': {'root': {'componentSummary': _value({'length': 0})}}, 'lists': {}} + return LoCo(path_response) @cache_utils.cache_output(cache_utils.CACHE_GENRES, identify_from_kwarg_name='genre_id', ignore_self_class=True) def req_loco_list_genre(self, genre_id): """Retrieve LoCo for the given genre""" LOG.debug('Requesting LoCo for genre {}', genre_id) - paths = ([['genres', genre_id, 'name'], - ['genres', genre_id, 'rw', 'componentSummary'], - ['genres', genre_id, 'rw', {'from': 0, 'to': 48}, 'componentSummary'], - # Titles of first 4 videos in each video list (needed only to show titles in the plot description) - ['genres', genre_id, 'rw', - {'from': 0, 'to': 48}, {'from': 0, 'to': 3}, 'reference', ['title', 'summary']]] + - # Art for the first video of each context list (needed only to add art to the menu item) - build_paths(['genres', genre_id, 'rw', {'from': 0, 'to': 48}, 0, 'reference'], ART_PARTIAL_PATHS) + - # IDs and names of sub-genres - [['genres', genre_id, 'subgenres', {'from': 0, 'to': 30}, ['id', 'name']]]) - call_args = {'paths': paths} - path_response = self.nfsession.path_request(**call_args) - return LoCo(path_response) + return self._req_browser_genre_loco(genre_id) def get_loco_list_id_by_context(self, context): """Return the dynamic video list ID for a LoCo context""" @@ -123,14 +1044,20 @@ def req_seasons(self, videoid, perpetual_range_start): raise InvalidVideoId(f'Cannot request season list for {videoid}') LOG.debug('Requesting the seasons list for show {}', videoid) call_args = { - 'paths': (build_paths(['videos', videoid.tvshowid], SEASONS_PARTIAL_PATHS)), - #+ build_paths(['videos', videoid.tvshowid], ART_PARTIAL_PATHS) + - #[['videos', videoid.tvshowid, 'componentSummary']]), + 'paths': (build_paths(['videos', videoid.tvshowid], SEASONS_PARTIAL_PATHS) + + build_paths(['videos', videoid.tvshowid], ART_PARTIAL_PATHS) + + [['videos', videoid.tvshowid, 'componentSummary']]), 'length_params': ['stdlist_wid', ['videos', videoid.tvshowid, 'seasonList']], 'perpetual_range_start': perpetual_range_start } - path_response = self.nfsession.perpetual_path_request(**call_args) - return SeasonList(videoid, path_response) + try: + path_response = self.nfsession.perpetual_path_request(**call_args) + return SeasonList(videoid, path_response) + except req_exceptions.HTTPError as exc: + if getattr(exc.response, 'status_code', None) != 404: + raise + LOG.warn('Falling back to GraphQL season selector for show {}', videoid.tvshowid) + return self._req_seasons_graphql(videoid) @cache_utils.cache_output(cache_utils.CACHE_COMMON, identify_from_kwarg_name='videoid', identify_append_from_kwarg_name='perpetual_range_start', ignore_self_class=True) @@ -140,25 +1067,796 @@ def req_episodes(self, videoid, perpetual_range_start=None): raise InvalidVideoId(f'Cannot request episode list for {videoid}') LOG.debug('Requesting episode list for {}', videoid) paths = ([['seasons', videoid.seasonid, 'summary']] + - [['videos', videoid.tvshowid, ['title', 'delivery']]] + - #[['seasons', videoid.seasonid, 'componentSummary']] + - build_paths(['seasons', videoid.seasonid, 'episodes', RANGE_PLACEHOLDER], EPISODES_PARTIAL_PATHS)) - # + build_paths(['videos', videoid.tvshowid], ART_PARTIAL_PATHS + [[['title', 'delivery']]])) + [['seasons', videoid.seasonid, 'componentSummary']] + + build_paths(['seasons', videoid.seasonid, 'episodes', RANGE_PLACEHOLDER], EPISODES_PARTIAL_PATHS) + + build_paths(['videos', videoid.tvshowid], ART_PARTIAL_PATHS + [[['title', 'delivery']]])) call_args = { 'paths': paths, 'length_params': ['stdlist_wid', ['seasons', videoid.seasonid, 'episodes']], 'perpetual_range_start': perpetual_range_start } - path_response = self.nfsession.perpetual_path_request(**call_args) - return EpisodeList(videoid, path_response) + try: + path_response = self.nfsession.perpetual_path_request(**call_args) + return EpisodeList(videoid, path_response) + except req_exceptions.HTTPError as exc: + if getattr(exc.response, 'status_code', None) != 404: + raise + LOG.warn('Falling back to GraphQL episode selector for season {}', videoid.seasonid) + return self._req_episodes_graphql(videoid) + + def _post_graphql(self, operation_name, variables, operation_id): + payload = { + 'operationName': operation_name, + 'variables': variables, + 'extensions': {'persistedQuery': {'id': operation_id, 'version': 102}} + } + response = self.nfsession.session.post( + GRAPHQL_URL, + json=payload, + headers=_graphql_headers(), + timeout=8) + response.raise_for_status() + return response.json()['data'] + + def _req_seasons_graphql(self, videoid): + data = self._post_graphql( + 'PreviewModalEpisodeSelector', + {'showId': int(videoid.tvshowid), 'seasonCount': 50}, + GRAPHQL_OP_SEASONS) + show_data = data['videos'][0] + edges = show_data['seasons']['edges'] + seasons = OrderedDict( + _season_node_to_item(edge.get('node') or edge, index) + for index, edge in enumerate(edges)) + show_title = self._metadata_show_title(videoid) or show_data.get('title') or str(videoid.tvshowid) + tvshow = { + 'title': _value(show_title), + 'delivery': _value({}), + 'seasonList': {'summary': _value({'length': len(seasons)})} + } + return SimpleNamespace( + perpetual_range_selector=None, + data={'videos': {videoid.tvshowid: tvshow}, 'seasons': seasons}, + videoid=videoid, + artitem=tvshow, + tvshow=tvshow, + seasons=seasons) + + def _metadata_show_title(self, videoid): + try: + metadata_data = self.nfsession.get_safe( + endpoint='metadata', + params={'movieid': videoid.tvshowid, '_': int(time.time() * 1000)}) + return metadata_data['video'].get('title') or '' + except (MetadataNotAvailable, KeyError, TypeError, req_exceptions.RequestException): + return '' + + def _metadata_episodes_by_id(self, videoid): + try: + metadata_data = self.nfsession.get_safe( + endpoint='metadata', + params={'movieid': videoid.tvshowid, '_': int(time.time() * 1000)}) + show_metadata = metadata_data['video'] + except (MetadataNotAvailable, KeyError, TypeError, req_exceptions.RequestException): + return {} + episodes = {} + for season in show_metadata.get('seasons', []): + if str(season.get('id')) != videoid.seasonid: + continue + for episode in season.get('episodes', []): + episodes[str(episode.get('id'))] = episode + break + return episodes + + + def _req_episodes_graphql(self, videoid): + data = self._post_graphql( + 'PreviewModalEpisodeSelectorSeasonEpisodes', + { + 'seasonId': int(videoid.seasonid), + 'count': 50, + 'opaqueImageFormat': 'JPG', + 'artworkContext': {} + }, + GRAPHQL_OP_EPISODES) + season_data = data['videos'][0] + season_number = season_data.get('number') + edges = season_data['episodes']['edges'] + metadata_by_id = self._metadata_episodes_by_id(videoid) + episodes = OrderedDict( + _episode_node_to_item(edge.get('node') or edge, season_number, + metadata_by_id.get(str((edge.get('node') or edge).get('videoId')))) + for edge in edges) + show_title = self._metadata_show_title(videoid) or str(videoid.tvshowid) + tvshow = { + 'title': _value(show_title), + 'delivery': _value({}) + } + season = { + 'summary': _summary(videoid.seasonid, season_data.get('title') or '', 'season', season_number, len(episodes)), + 'title': _value(season_data.get('title') or '') + } + path_response = {'videos': {videoid.tvshowid: tvshow}, 'seasons': {videoid.seasonid: season}, + 'episodes': episodes} + for episode_id, episode in episodes.items(): + normalize_metadata_references(path_response, episode_id, metadata_by_id.get(str(episode_id)), episode) + return SimpleNamespace( + perpetual_range_selector=None, + data=path_response, + videoid=videoid, + tvshow=tvshow, + season=season, + episodes=episodes) + + def _browse_html_and_auth_url(self): + browse_html = self.nfsession.get_safe('browse') + api_data = self.nfsession.website_extract_session_data(browse_html) + self.nfsession.auth_url = api_data['auth_url'] + browse_text = browse_html.decode('utf-8', 'replace') if isinstance(browse_html, bytes) else browse_html + return browse_text, api_data['auth_url'] + + def _get_current_loco_root_id(self): + browse_html, auth_url = self._browse_html_and_auth_url() + match = LOCO_ROOT_ID_RE.search(browse_html) + if match: + return match.group(0), auth_url + root_id = self._probe_current_loco_root_id(self._loco_root_candidates(browse_html), auth_url) + if not root_id: + raise InvalidVideoListTypeError('No current LoCo root id found in browse page') + return root_id, auth_url + + def _loco_root_candidates(self, browse_html): + seen = set() + for match in LOCO_ROOT_CANDIDATE_RE.finditer(browse_html): + candidate = match.group(0) + if candidate in seen: + continue + seen.add(candidate) + yield candidate + + def _probe_current_loco_root_id(self, candidates, auth_url): + for candidate in candidates: + try: + path_response = self._post_current_loco_paths( + [['locos', candidate, 'componentSummary']], auth_url) + except req_exceptions.RequestException: + continue + root_data = path_response.get('locos', {}).get(candidate) + if isinstance(root_data, dict) and root_data.get('componentSummary', {}).get('value'): + LOG.warn('Using probed current LoCo root candidate from browse page') + return candidate + return None + + def _current_loco_paths(self, root_id): + return ([ + ['locos', root_id, 'componentSummary'], + ['locos', root_id, LOCO_ROW_RANGE, 'componentSummary'], + ['locos', root_id, LOCO_ROW_RANGE, 'page', 0, LOCO_PAGE_RANGE, 'itemSummary'], + ['locos', root_id, LOCO_ROW_RANGE, 'page', 0, LOCO_PAGE_RANGE, 'reference', LOCO_REFERENCE_FIELDS] + ] + build_paths( + ['locos', root_id, LOCO_ROW_RANGE, 'page', 0, LOCO_PAGE_RANGE, 'reference'], + ART_PARTIAL_PATHS)) + + def _post_current_loco_paths(self, paths, auth_url): + self.nfsession.auth_url = auth_url + return self._post_browser_path_evaluator(paths, 'https://www.netflix.com/browse') + + def _post_browser_path_evaluator_with_fallback(self, paths, fallback_paths, referer, description): + try: + return self._post_browser_path_evaluator(paths, referer) + except req_exceptions.HTTPError as exc: + status_code = getattr(exc.response, 'status_code', None) + if status_code not in (404, 412): + raise + LOG.warn('{} metadata fields request returned {}; retrying light fields', + description, status_code) + return self._post_browser_path_evaluator(fallback_paths, referer) + + def _post_browser_path_evaluator(self, paths, referer): + api_url = G.LOCAL_DB.get_value( + 'api_endpoint_url', + 'https://www.netflix.com/nq/website/memberapi/release', + table=TABLE_SESSION) + form_data = [('path', json.dumps(path, separators=(',', ':'))) for path in paths] + form_data.append(('authURL', self.nfsession.auth_url)) + response = self.nfsession.session.post( + f'{api_url}/pathEvaluator', + params={ + 'webp': 'false', + 'drmSystem': 'widevine', + 'isVolatileBillboardsEnabled': 'true', + 'isTop10Supported': 'true', + 'hasVideoMerchInBob': 'false', + 'hasVideoMerchInJaw': 'false', + 'falcor_server': '0.1.0', + 'withSize': 'true', + 'materialize': 'true', + 'original_path': '/shakti/mre/pathEvaluator' + }, + data=urlencode(form_data), + headers={ + 'Accept': '*/*', + 'Content-Type': 'application/x-www-form-urlencoded', + 'Origin': 'https://www.netflix.com', + 'Referer': referer, + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'x-netflix.nq.stack': 'prod', + 'x-netflix.request.client.context': 'www.netflix.com', + 'x-netflix.request.client.user.guid': G.LOCAL_DB.get_active_profile_guid() + }, + timeout=8) + response.raise_for_status() + path_response = response.json()['jsonGraph'] + _normalize_browser_video_fields(path_response) + return path_response + + def _browser_loco_paths(self, root_path, include_genre_paths=False, include_full_rows=False, + include_metadata=False): + paths = [ + root_path + [['componentSummary', 'debugRequest']], + root_path + [BROWSER_LOCO_ROW_KEYS, 'componentSummary'], + root_path + ['meta', ['responseExpiration', 'statusCode']], + root_path + [0, 0, 'itemSummary'], + root_path + [0, 0, 'reference', BROWSER_LOCO_SUMMARY_FIELDS], + root_path + [0, 0, 'reference', 'current', BROWSER_LOCO_CURRENT_FIELDS], + root_path + [0, 'page', 0, LOCO_PAGE_RANGE, 'itemSummary'], + root_path + [BROWSER_LOCO_OTHER_ROW_KEYS, 'page', 0, LOCO_PAGE_RANGE, 'itemSummary'], + root_path + ['continueWatching', 'page', 0, LOCO_PAGE_RANGE, 'reference', 'current', + BROWSER_LOCO_CONTINUE_FIELDS] + ] + paths.extend(_browser_reference_paths( + root_path + [BROWSER_LOCO_ROW_KEYS, 'page', 0, LOCO_PAGE_RANGE, 'reference'])) + if include_full_rows: + paths.append(root_path + [BROWSER_LOCO_ROW_KEYS, BROWSER_LOCO_DIRECT_RANGE, 'itemSummary']) + paths.extend(_browser_reference_paths( + root_path + [BROWSER_LOCO_ROW_KEYS, BROWSER_LOCO_DIRECT_RANGE, 'reference'], + include_metadata=include_metadata)) + if include_genre_paths: + paths.insert(0, root_path[:-1] + [['name', 'trackIds']]) + return paths + + def _browser_video_list_paths(self, list_id, include_metadata=False): + paths = [ + ['lists', list_id, ['componentSummary', 'debugRequest']], + ['lists', list_id, 'page', 0, LOCO_PAGE_RANGE, 'itemSummary'] + ] + paths.extend(_browser_reference_paths(['lists', list_id, 'page', 0, LOCO_PAGE_RANGE, 'reference'], + include_metadata=include_metadata)) + return paths + + def _browser_video_list_full_paths(self, list_id, include_metadata=False): + paths = [ + ['lists', list_id, ['componentSummary', 'debugRequest']], + ['lists', list_id, BROWSER_LOCO_DIRECT_RANGE, 'itemSummary'] + ] + paths.extend(_browser_reference_paths(['lists', list_id, BROWSER_LOCO_DIRECT_RANGE, 'reference'], + include_metadata=include_metadata)) + return paths + + def _req_browser_lolomo_category(self, category_name): + self._browse_html_and_auth_url() + path_response = self._post_browser_path_evaluator( + self._browser_loco_paths(['lolomoByCategory', category_name]), + 'https://www.netflix.com/latest') + return LoLoMoCategory(path_response) + + def _req_browser_genre_loco(self, genre_id): + self._browse_html_and_auth_url() + path_response = self._post_browser_path_evaluator( + self._browser_loco_paths(['genres', int(genre_id), 'rw'], include_genre_paths=True), + f'https://www.netflix.com/browse/genre/{genre_id}') + return LoCo(path_response) + + def _browser_video_list_by_id(self, list_id): + self._browse_html_and_auth_url() + path_response = self._post_browser_path_evaluator_with_fallback( + self._browser_video_list_paths(str(list_id), include_metadata=True), + self._browser_video_list_paths(str(list_id)), + 'https://www.netflix.com/browse', + f'Browser list {list_id}') + return VideoList(path_response, str(list_id)) + + def _browser_mylist_loco_response(self, root_id, auth_url, row_range): + return self._post_current_loco_paths([ + ['locos', root_id, 'componentSummary'], + ['locos', root_id, row_range, 'componentSummary'] + ], auth_url) + + def _loco_row_key_for_list(self, root_response, root_id, list_id): + root_data = root_response.get('locos', {}).get(root_id, {}) + for row_key, row_data in root_data.items(): + if row_key == 'componentSummary' or not isinstance(row_data, dict): + continue + row_ref = row_data.get('value') + if isinstance(row_ref, list) and len(row_ref) > 1 and str(row_ref[1]) == str(list_id): + return int(row_key) if str(row_key).isdigit() else row_key + return None + + def _browser_mylist_list_info(self, root_id, auth_url): + for row_range in (BROWSER_LOCO_HOME_VISIBLE_RANGE, BROWSER_LOCO_HOME_ROW_RANGE, LOCO_ROW_RANGE): + try: + root_response = self._browser_mylist_loco_response(root_id, auth_url, row_range) + except req_exceptions.HTTPError as exc: + if getattr(exc.response, 'status_code', None) not in (404, 412): + raise + LOG.warn('My List queue lookup range {} returned {}; trying another range', + row_range, exc.response.status_code) + continue + list_id, _video_list = LoCo(root_response).find_by_context('queue') + if list_id: + return str(list_id), self._loco_row_key_for_list(root_response, root_id, list_id) + raise InvalidVideoListTypeError('No current LoCo My List queue available') + + def _browser_mylist_loco_row_paths(self, root_id, row_key, use_direct_range, include_metadata=False): + item_range = BROWSER_LOCO_DIRECT_RANGE if use_direct_range else LOCO_PAGE_RANGE + row_path = ['locos', root_id, row_key] + if use_direct_range: + return [ + row_path + ['componentSummary'], + row_path + [item_range, 'itemSummary'], + *_browser_reference_paths(row_path + [item_range, 'reference'], + include_metadata=include_metadata) + ] + return [ + row_path + ['componentSummary'], + row_path + ['page', 0, item_range, 'itemSummary'], + *_browser_reference_paths(row_path + ['page', 0, item_range, 'reference'], + include_metadata=include_metadata) + ] + + def _browser_mylist_loco_video_list(self, root_id, row_key, list_id, auth_url): + for use_direct_range in (True, False): + metadata_paths = self._browser_mylist_loco_row_paths(root_id, row_key, use_direct_range, + include_metadata=True) + light_paths = self._browser_mylist_loco_row_paths(root_id, row_key, use_direct_range) + try: + try: + path_response = self._post_current_loco_paths(metadata_paths, auth_url) + except req_exceptions.HTTPError as exc: + status_code = getattr(exc.response, 'status_code', None) + if status_code not in (404, 412): + raise + LOG.warn('My List LoCo metadata fields request returned {}; retrying light fields', + status_code) + path_response = self._post_current_loco_paths(light_paths, auth_url) + if str(list_id) in path_response.get('lists', {}): + return VideoList(path_response, str(list_id)) + except req_exceptions.HTTPError as exc: + if getattr(exc.response, 'status_code', None) not in (404, 412): + raise + LOG.warn('My List LoCo row content request returned {}; trying fallback path', + exc.response.status_code) + raise InvalidVideoListTypeError('No current LoCo My List content available') + + def _browser_mylist_direct_video_list(self, list_id, auth_url): + for paths, fallback_paths in ( + (self._browser_video_list_full_paths(str(list_id), include_metadata=True), + self._browser_video_list_full_paths(str(list_id))), + (self._browser_video_list_paths(str(list_id), include_metadata=True), + self._browser_video_list_paths(str(list_id)))): + try: + try: + path_response = self._post_current_loco_paths(paths, auth_url) + except req_exceptions.HTTPError as exc: + status_code = getattr(exc.response, 'status_code', None) + if status_code not in (404, 412): + raise + LOG.warn('My List direct metadata fields request returned {}; retrying light fields', + status_code) + path_response = self._post_current_loco_paths(fallback_paths, auth_url) + return VideoList(path_response, str(list_id)) + except req_exceptions.HTTPError as exc: + if getattr(exc.response, 'status_code', None) not in (404, 412): + raise + LOG.warn('My List direct list content request returned {}; trying fallback path', + exc.response.status_code) + raise InvalidVideoListTypeError('No current direct My List content available') + + def _browser_mylist_video_list(self): + self._browse_html_and_auth_url() + path_response = self._post_browser_path_evaluator([ + ['mylist', ['id', 'listId', 'name', 'requestId', 'trackIds']], + ['mylist', BROWSER_MYLIST_RANGE, BROWSER_MYLIST_FIELDS] + ], 'https://www.netflix.com/browse/my-list') + mylist_data = path_response.get('mylist') + if not isinstance(mylist_data, dict): + raise InvalidVideoListTypeError('No browser My List data available') + path_response['lists'] = {'mylist': mylist_data} + _normalize_browser_video_fields(path_response) + video_list = VideoList(path_response, 'mylist') + for video in video_list.videos.values(): + video.setdefault('queue', _value({})) + video['queue'].setdefault('value', {})['inQueue'] = True + return video_list + + def _enrich_browser_video_list_metadata(self, path_response, list_id): + videos = path_response.get('videos', {}) + for video_id in _browser_list_video_ids(path_response, list_id): + video_key = next((key for key in videos if str(key) == video_id), None) + if video_key is None: + continue + video = videos.get(video_key) + if not isinstance(video, dict): + continue + fallback_art = common.get_path_safe( + ['itemSummary', 'value', 'boxArt', 'url'], video) + poster_art = common.get_path_safe( + ['boxarts', ART_SIZE_POSTER, 'jpg', 'value', 'url'], video) + synopsis = (video.get('synopsis', {}).get('value') or + video.get('regularSynopsis', {}).get('value')) + if synopsis and poster_art and poster_art != fallback_art: + continue + try: + metadata_video = self.nfsession._metadata( # pylint: disable=protected-access + video_id=common.VideoId(videoid=video_id)) + except (MetadataNotAvailable, KeyError, TypeError, req_exceptions.RequestException): + LOG.warn('LoLoMo metadata enrichment skipped for video {}', video_id) + continue + videos[video_key] = _merge_search_metadata_video(video, metadata_video) + + def _browser_lolomo_video_list_by_id(self, category_name, list_id): + self._browse_html_and_auth_url() + path_response = self._post_browser_path_evaluator( + self._browser_loco_paths(['lolomoByCategory', category_name], include_full_rows=True), + 'https://www.netflix.com/latest') + if str(list_id) not in path_response.get('lists', {}): + raise InvalidVideoListTypeError(f'No LoLoMo category list with id {list_id}') + return VideoList(path_response, str(list_id)) + + def _browser_genre_video_list_by_id(self, genre_id, list_id): + self._browse_html_and_auth_url() + path_response = self._post_browser_path_evaluator_with_fallback( + self._browser_loco_paths(['genres', int(genre_id), 'rw'], include_genre_paths=True, + include_full_rows=True, include_metadata=True), + self._browser_loco_paths(['genres', int(genre_id), 'rw'], include_genre_paths=True, + include_full_rows=True), + f'https://www.netflix.com/browse/genre/{genre_id}', + f'Genre {genre_id}') + if str(list_id) not in path_response.get('lists', {}): + raise InvalidVideoListTypeError(f'No genre list with id {list_id}') + return VideoList(path_response, str(list_id)) + + def _browser_continue_watching_loco_response(self, root_id, auth_url, row_range): + return self._post_current_loco_paths([ + ['locos', root_id, row_range, 'componentSummary'], + ['locos', root_id, row_range, 'page', 0, LOCO_PAGE_RANGE, 'itemSummary'], + *_browser_reference_paths(['locos', root_id, row_range, 'page', 0, LOCO_PAGE_RANGE, 'reference']) + ], auth_url) + + def _continue_watching_list_id(self, root_response): + candidates = [] + for candidate_id, list_data in root_response.get('lists', {}).items(): + summary = list_data.get('componentSummary', {}).get('value', {}) + if summary.get('context') != 'continueWatching': + continue + length = summary.get('length') or 0 + materialized_items = sum(1 for key, value in list_data.items() + if str(key).isdigit() and isinstance(value, dict)) + candidates.append((length, materialized_items, candidate_id)) + return max(candidates)[2] if candidates else None + + def _browser_continue_watching_direct_response(self, list_id): + return self._post_browser_path_evaluator([ + ['lists', list_id, ['componentSummary', 'debugRequest']], + ['lists', list_id, BROWSER_LOCO_DIRECT_RANGE, 'itemSummary'], + *_browser_reference_paths(['lists', list_id, BROWSER_LOCO_DIRECT_RANGE, 'reference']), + ['lists', list_id, BROWSER_LOCO_DIRECT_RANGE, 'reference', 'current', + BROWSER_LOCO_CONTINUE_FIELDS] + ], 'https://www.netflix.com/browse') + + def _browser_continue_watching_list(self): + try: + return self._browser_continue_watching_graphql_list() + except (InvalidVideoListTypeError, WebsiteParsingError, KeyError, TypeError, + ValueError, req_exceptions.RequestException) as exc: + LOG.warn('GraphQL Continue Watching lookup failed ({}); using genre fallback', + type(exc).__name__) + return self._browser_continue_watching_genre_fallback() + + def _browser_continue_watching_graphql_list(self): + graphql_data, section, connection = self._browser_graphql_carousel_section( + self._continue_watching_graphql_section) + videos = OrderedDict() + self._append_continue_watching_graphql_edges( + videos, graphql_data, _iter_graphql_edges(connection)) + page_info = connection.get('pageInfo') or {} + while page_info.get('hasNextPage') and page_info.get('endCursor'): + data = self._post_graphql( + 'CarouselPage', + _carousel_graphql_variables(section.get('_id') or section['id'], page_info['endCursor']), + GRAPHQL_OP_CAROUSEL_PAGE) + next_section = data.get('node') or {} + next_connection = next_section.get('entities') or {} + previous_count = len(videos) + self._append_continue_watching_graphql_edges( + videos, {}, _iter_graphql_edges(next_connection)) + page_info = next_connection.get('pageInfo') or {} + if len(videos) == previous_count: + break + if not videos: + raise InvalidVideoListTypeError('No GraphQL Continue Watching videos available') + return CustomVideoList({'videos': videos}) + + def _browser_top_picks_list(self): + """Return the personalized Top Picks carousel from the active home page.""" + graphql_data, _section, connection = self._browser_graphql_carousel_section( + self._top_picks_graphql_section) + videos = OrderedDict() + self._append_standard_graphql_edges( + videos, graphql_data, _iter_graphql_edges(connection)) + if not videos: + raise InvalidVideoListTypeError('No GraphQL Top Picks videos available') + LOG.debug('GraphQL Top Picks returned {} personalized videos', len(videos)) + return CustomVideoList({'videos': videos}) + + def _browser_graphql_carousel_section(self, section_resolver): + browse_html = self.nfsession.get_safe('browse') + graphql_data = self._browser_graphql_data(browse_html) + try: + section, connection = section_resolver(graphql_data) + except InvalidVideoListTypeError as section_error: + try: + browse_html = self._active_profile_browse_html(browse_html) + except InvalidVideoListTypeError: + raise section_error + graphql_data = self._browser_graphql_data(browse_html) + section, connection = section_resolver(graphql_data) + return graphql_data, section, connection + + def _browser_graphql_data(self, browse_html): + api_data = self.nfsession.website_extract_session_data(browse_html) + self.nfsession.auth_url = api_data['auth_url'] + react_context = website.extract_json(browse_html, 'reactContext') + return _title_page_graphql_data(browse_html, react_context) + + def _active_profile_browse_html(self, profile_gate_html): + parser = _ActiveProfileLinkParser(G.LOCAL_DB.get_active_profile_guid()) + parser.feed(profile_gate_html.decode('utf-8', 'replace') + if isinstance(profile_gate_html, bytes) else str(profile_gate_html)) + if not parser.href: + raise InvalidVideoListTypeError('No active profile switch link available') + response = self.nfsession.session.get( + urljoin('https://www.netflix.com/browse', parser.href), + headers={ + 'Accept': 'text/html,application/xhtml+xml,application/xml', + 'Referer': 'https://www.netflix.com/browse', + 'User-Agent': common.get_user_agent(enable_android_mediaflag_fix=True) + }, + timeout=8) + response.raise_for_status() + return response.content + + @staticmethod + def _continue_watching_graphql_section(graphql_data): + for section in graphql_data.values(): + if not isinstance(section, dict) or section.get('__typename') != 'PinotCarouselSection': + continue + connection = _graphql_ref_node(graphql_data, section.get('entities')) + if not isinstance(connection, dict): + continue + for edge in _iter_graphql_edges(connection): + edge_data = _graphql_ref_node(graphql_data, edge) + node = _graphql_ref_node(graphql_data, (edge_data or {}).get('node')) + if isinstance(node, dict) and node.get('__typename') == 'PinotContinueWatchingEntityTreatment': + return section, connection + raise InvalidVideoListTypeError('No GraphQL Continue Watching section available') + + @staticmethod + def _top_picks_graphql_section(graphql_data): + labels = { + TOP_PICKS_SECTION_LABEL, + str(common.get_local_string(30169) or '').casefold() + } + for section in graphql_data.values(): + if not isinstance(section, dict) or section.get('__typename') != 'PinotCarouselSection': + continue + label = str(section.get('displayString') or '').casefold() + if not any(candidate and candidate in label for candidate in labels): + continue + connection = _graphql_ref_node(graphql_data, section.get('entities')) + if isinstance(connection, dict): + return section, connection + raise InvalidVideoListTypeError('No personalized GraphQL Top Picks section available') + + @staticmethod + def _continue_watching_graphql_node(graphql_data, edge): + edge_data = _graphql_ref_node(graphql_data, edge) if graphql_data else edge + node = ((edge_data or {}).get('node') or {}) + node = _graphql_ref_node(graphql_data, node) if graphql_data else node + if not isinstance(node, dict): + return None + entity = node.get('unifiedEntity') or {} + entity = _graphql_ref_node(graphql_data, entity) if graphql_data else entity + artwork_context = node.get('contextualArtwork') or {} + artwork_context = (_graphql_ref_node(graphql_data, artwork_context) + if graphql_data else artwork_context) + artwork = {} + if isinstance(artwork_context, dict): + artwork_value = next( + (value for key, value in artwork_context.items() if key == 'artwork' or key.startswith('artwork(')), + {}) + artwork = (_graphql_ref_node(graphql_data, artwork_value) + if graphql_data else artwork_value) + return { + 'displayString': node.get('displayString'), + 'unifiedEntity': entity or {}, + 'contextualArtwork': {'artwork': artwork or {}} + } + + def _append_continue_watching_graphql_edges(self, videos, graphql_data, edges): + for edge in edges: + node = self._continue_watching_graphql_node(graphql_data, edge) + item_data = _search_graphql_node_to_item(node or {}) + if not item_data: + continue + video_id, item = item_data + entity = node.get('unifiedEntity') or {} + progress_entity = entity.get('currentEpisode') or entity + if graphql_data: + progress_entity = _graphql_ref_node(graphql_data, progress_entity) or progress_entity + bookmark = progress_entity.get('bookmark') or {} + if graphql_data: + bookmark = _graphql_ref_node(graphql_data, bookmark) or bookmark + item['bookmarkPosition'] = _value(bookmark.get('position', 0)) + item['runtime'] = _value(progress_entity.get('runtimeSec') or entity.get('runtimeSec') or 0) + videos[video_id] = item + + def _append_standard_graphql_edges(self, videos, graphql_data, edges): + for edge in edges: + node = self._continue_watching_graphql_node(graphql_data, edge) + item_data = _search_graphql_node_to_item(node or {}) + if not item_data: + continue + video_id, item = item_data + videos.setdefault(video_id, item) + + def _browser_continue_watching_loco_list(self): + try: + root_id, auth_url = self._get_current_loco_root_id() + except InvalidVideoListTypeError: + return self._browser_continue_watching_genre_fallback() + root_response = self._browser_continue_watching_loco_response( + root_id, auth_url, BROWSER_LOCO_HOME_ROW_RANGE) + list_id = self._continue_watching_list_id(root_response) + if not list_id: + root_response = self._browser_continue_watching_loco_response( + root_id, auth_url, BROWSER_LOCO_HOME_VISIBLE_RANGE) + list_id = self._continue_watching_list_id(root_response) + if not list_id: + raise InvalidVideoListTypeError('No current home Continue Watching list available') + try: + direct_response = self._browser_continue_watching_direct_response(list_id) + root_response.setdefault('lists', {}).setdefault(list_id, {}).update( + direct_response.get('lists', {}).get(list_id, {})) + root_response.setdefault('videos', {}).update(direct_response.get('videos', {})) + _normalize_browser_video_fields(root_response) + except req_exceptions.HTTPError as exc: + if getattr(exc.response, 'status_code', None) not in (404, 412): + raise + LOG.warn('Continue Watching direct list range returned {}; using home row data', + exc.response.status_code) + length = root_response['lists'][list_id].get('componentSummary', {}).get('value', {}).get('length', 0) + if length > BROWSER_LOCO_CONTINUE_LAZY_RANGE['from']: + try: + lazy_response = self._post_browser_path_evaluator([ + ['lists', list_id, BROWSER_LOCO_CONTINUE_LAZY_RANGE, 'itemSummary'], + *_browser_reference_paths(['lists', list_id, BROWSER_LOCO_CONTINUE_LAZY_RANGE, 'reference']), + ['lists', list_id, BROWSER_LOCO_CONTINUE_LAZY_RANGE, 'reference', 'current', + BROWSER_LOCO_CONTINUE_FIELDS] + ], 'https://www.netflix.com/browse') + root_response.setdefault('lists', {}).setdefault(list_id, {}).update( + lazy_response.get('lists', {}).get(list_id, {})) + root_response.setdefault('videos', {}).update(lazy_response.get('videos', {})) + _normalize_browser_video_fields(root_response) + except req_exceptions.HTTPError as exc: + if getattr(exc.response, 'status_code', None) not in (404, 412): + raise + LOG.warn('Continue Watching lazy range returned {}; using available row data', + exc.response.status_code) + return VideoList(root_response, str(list_id)) + + def _browser_continue_watching_genre_fallback(self): + LOG.warn('Home LoCo discovery failed for Continue Watching; using genre fallback') + try: + loco_list = self.req_loco_list_genre('1592210') + for list_id, video_list in loco_list.lists.items(): + if video_list.get('context') != 'continueWatching': + continue + try: + return self._browser_genre_video_list_by_id('1592210', list_id) + except Exception: # pylint: disable=broad-except + LOG.warn('Using materialized Continue Watching genre row after browser list lookup failed') + return video_list + except Exception: # pylint: disable=broad-except + LOG.warn('Continue Watching genre fallback failed after home LoCo discovery failure') + return CustomVideoList({'videos': {}}) + + def _first_loco_video_list(self, loco): + for _list_id, video_list in loco.lists.items(): + if video_list.videos: + return video_list + return next(iter(loco.lists.values())) + + def _first_full_browser_genre_video_list(self, genre_id): + loco = self._req_browser_genre_loco(genre_id) + first_list_id = None + first_video_list = None + for list_id, video_list in loco.lists.items(): + if first_list_id is None: + first_list_id = list_id + first_video_list = video_list + if video_list.videos: + first_list_id = list_id + first_video_list = video_list + break + if first_list_id is None: + raise InvalidVideoListTypeError(f'No browser genre rows available for {genre_id}') + try: + return self._browser_genre_video_list_by_id(genre_id, first_list_id) + except Exception as exc: # pylint: disable=broad-except + LOG.warn('Using materialized genre preview row after full row lookup failed: {}', exc) + return first_video_list + + def _req_current_loco_root_data(self): + root_id, auth_url = self._get_current_loco_root_id() + return self._post_current_loco_paths(self._current_loco_paths(root_id), auth_url) + + def _current_loco_list_by_context(self, context): + loco = LoCo(self._req_current_loco_root_data()) + list_id, video_list = loco.find_by_context(context) + if not list_id: + category_contexts = LOCO_CATEGORY_CONTEXTS.get('comingSoon', ()) + if context in category_contexts: + for _list_id, summary, category_video_list in self.req_lolomo_category('comingSoon').lists(): + if summary.get('context') == context: + return category_video_list + raise InvalidVideoListTypeError(f'No current LoCo list with context {context} available') + return video_list + + def _current_loco_list_by_id(self, list_id): + loco = LoCo(self._req_current_loco_root_data()) + if str(list_id) not in loco.data.get('lists', {}): + raise InvalidVideoListTypeError(f'No current LoCo list with id {list_id} available') + return VideoList(loco.data, str(list_id)) + + def _current_lolomo_category(self, category_name): + contexts = LOCO_CATEGORY_CONTEXTS.get(category_name) + if not contexts: + raise InvalidVideoListTypeError(f'No current LoCo fallback for category {category_name}') + loco = LoCo(self._req_current_loco_root_data()) + lists = OrderedDict( + (list_id, list_data) + for list_id, list_data in loco.data.get('lists', {}).items() + if list_data.get('componentSummary', {}).get('value', {}).get('context') in contexts) + root_id = loco.id + root = OrderedDict() + root['componentSummary'] = _value({'length': len(lists)}) + for index, list_id in enumerate(lists): + root[index] = { + 'reference': _value(['lists', list_id]), + 'itemSummary': _value({'id': list_id}) + } + return LoLoMoCategory({ + 'locos': {root_id: root}, + 'lists': lists, + 'videos': loco.data.get('videos', {}) + }) + @cache_utils.cache_output(cache_utils.CACHE_COMMON, identify_append_from_kwarg_name='perpetual_range_start', ignore_self_class=True) - def req_video_list(self, list_id, perpetual_range_start=None): + def req_video_list(self, list_id, perpetual_range_start=None, menu_data=None): """Retrieve a video list""" # Some of this type of request have results fixed at ~40 from netflix # The 'length' tag never return to the actual total count of the elements LOG.debug('Requesting video list {}', list_id) + browser_genre_id = str((menu_data or {}).get('browser_genre_id') or '') + if browser_genre_id: + if str(list_id) == browser_genre_id: + return self._first_full_browser_genre_video_list(browser_genre_id) + return self._browser_genre_video_list_by_id(browser_genre_id, list_id) paths = (build_paths(['lists', list_id, RANGE_PLACEHOLDER, 'reference'], VIDEO_LIST_PARTIAL_PATHS) + [['lists', list_id, 'componentSummary']]) call_args = { @@ -166,7 +1864,20 @@ def req_video_list(self, list_id, perpetual_range_start=None): 'length_params': ['stdlist', ['lists', list_id]], 'perpetual_range_start': perpetual_range_start } - path_response = self.nfsession.perpetual_path_request(**call_args) + try: + path_response = self.nfsession.perpetual_path_request(**call_args) + except req_exceptions.HTTPError as exc: + if getattr(exc.response, 'status_code', None) != 404: + raise + initial_menu_id = (menu_data or {}).get('initial_menu_id') + if initial_menu_id in ('newAndPopular', 'recommendations'): + LOG.warn('Falling back to browser-shaped LoLoMo category list {} after pathEvaluator 404', list_id) + return self._browser_lolomo_video_list_by_id('comingSoon', list_id) + LOG.warn('Falling back to browser-shaped list {} after pathEvaluator 404', list_id) + try: + return self._browser_video_list_by_id(list_id) + except req_exceptions.HTTPError: + return self._current_loco_list_by_id(list_id) return VideoList(path_response) @cache_utils.cache_output(cache_utils.CACHE_COMMON, identify_from_kwarg_name='context_id', @@ -176,6 +1887,19 @@ def req_video_list_sorted(self, context_name, context_id=None, perpetual_range_s # This type of request allows to obtain more than ~40 results LOG.debug('Requesting video list sorted for context name: "{}", context id: "{}"', context_name, context_id) + if context_name == 'mylist': + try: + return self._browser_mylist_video_list() + except InvalidVideoListTypeError: + LOG.warn('Returning empty My List after current queue lookup failed') + return CustomVideoList({'videos': {}}) + except req_exceptions.HTTPError as exc: + if getattr(exc.response, 'status_code', None) not in (404, 412): + raise + LOG.warn('Returning empty My List after browser-shaped request returned {}', + exc.response.status_code) + return CustomVideoList({'videos': {}}) + base_path = [context_name] response_type = 'stdlist' if context_id: @@ -196,7 +1920,17 @@ def req_video_list_sorted(self, context_name, context_id=None, perpetual_range_s paths = (build_paths(_base_path, VIDEO_LIST_PARTIAL_PATHS) + [base_path[:-1] + [['id', 'name', 'requestId', 'trackIds']]]) - path_response = self.nfsession.perpetual_path_request(paths, [response_type, base_path], perpetual_range_start) + try: + path_response = self.nfsession.perpetual_path_request(paths, [response_type, base_path], perpetual_range_start) + except req_exceptions.HTTPError as exc: + status_code = getattr(exc.response, 'status_code', None) + if status_code not in (404, 412): + raise + context = SORTED_LIST_CONTEXT_FALLBACKS.get((context_name, str(context_id))) + if context_name != 'genres' and not context: + raise + LOG.warn('Falling back to browser-shaped genre {} after pathEvaluator {}', context_id, status_code) + return self._first_full_browser_genre_video_list(context_id) return VideoListSorted(path_response, context_name, context_id, req_sort_order_type) @@ -236,11 +1970,147 @@ def req_video_list_supplemental(self, videoid, supplemental_type): if videoid.mediatype not in (common.VideoId.SHOW, common.VideoId.MOVIE): raise InvalidVideoId(f'Cannot request video list supplemental for {videoid}') LOG.debug('Requesting video list supplemental of type "{}" for {}', supplemental_type, videoid) + if supplemental_type == SUPPLEMENTAL_TYPE_TRAILERS: + try: + trailer_list = self._req_video_list_supplemental_graphql(videoid) + if trailer_list.videos: + return trailer_list + LOG.warn('Website GraphQL returned no trailers for {}', videoid) + return trailer_list + except (KeyError, TypeError, ValueError, req_exceptions.RequestException) as exc: + LOG.warn('Website trailer collection lookup failed for {} ({}), trying title page fallback', + videoid, type(exc).__name__) + return self._req_video_list_supplemental_title_page(videoid) + path = build_paths( ['videos', videoid.value, supplemental_type, {"from": 0, "to": 35}], TRAILER_PARTIAL_PATHS ) - path_response = self.nfsession.path_request(path) - return VideoListSupplemental(path_response, 'videos', videoid.value, supplemental_type) + parent_metadata = {'loaded': False, 'value': None} + + def _get_parent_metadata(): + if not parent_metadata['loaded']: + parent_metadata['loaded'] = True + parent_metadata['value'] = self._metadata_for_video(videoid.value, 'Parent supplemental') + return parent_metadata['value'] + + def _inherit_parent_metadata(video_list): + metadata_video = _get_parent_metadata() + if metadata_video: + for supplemental_id, supplemental_video in video_list.videos.items(): + normalize_metadata_references(video_list.data, supplemental_id, metadata_video, supplemental_video) + return video_list + + def _empty_fallback(): + return SimpleNamespace( + perpetual_range_selector=None, + videos=OrderedDict(), + artitem=None, + contained_titles=[], + component_summary={}) + + def _title_page_fallback(): + trailer_list = self._req_video_list_supplemental_title_page(videoid) + return _inherit_parent_metadata(trailer_list) if trailer_list.videos else _empty_fallback() + try: + path_response = self.nfsession.path_request(path) + trailer_list = VideoListSupplemental(path_response, 'videos', videoid.value, supplemental_type) + if trailer_list.videos: + return _inherit_parent_metadata(trailer_list) + LOG.warn('Trailer supplemental response was empty for {}, trying title page fallback', videoid) + return _title_page_fallback() + except req_exceptions.HTTPError as exc: + if getattr(exc.response, 'status_code', None) != 404: + raise + LOG.warn('Trailer supplemental path returned 404 for {}, trying title page fallback', videoid) + return _title_page_fallback() + + def _req_video_list_supplemental_graphql(self, videoid): + """Load the website's ordered Trailers & More collection.""" + video_id = int(videoid.value) + detail_data = self._post_graphql( + 'DetailModal', + { + 'artworkContext': {}, + 'checkLinearChannel': True, + 'fetchPromoVideoOverride': False, + 'hasPromoVideoOverride': False, + 'isLiveEpisodic': False, + 'opaqueImageFormat': 'WEBP', + 'promoVideoId': 0, + 'textEvidenceUiContext': 'ODP', + 'transparentImageFormat': 'WEBP', + 'unifiedEntityId': f'Video:{video_id}', + 'videoId': video_id, + 'videoMerchContext': 'BROWSE', + 'videoMerchEnabled': False + }, + GRAPHQL_OP_DETAIL_MODAL) + entity = (detail_data.get('unifiedEntities') or [None])[0] or {} + edges = common.get_path_safe(['supplementalVideosList', 'edges'], entity, False, []) + trailer_ids = [] + for edge in edges: + node = edge.get('node') or edge + trailer_id = node.get('videoId') + if trailer_id and trailer_id not in trailer_ids: + trailer_ids.append(trailer_id) + if not trailer_ids: + return self._empty_supplemental_list() + + trailer_data = self._post_graphql( + 'DetailModalTrailers', + { + 'artworkContext': {}, + 'opaqueImageFormat': 'WEBP', + 'videoIds': trailer_ids + }, + GRAPHQL_OP_DETAIL_MODAL_TRAILERS) + nodes_by_id = { + str(node.get('videoId')): node + for node in trailer_data.get('videos') or [] + if isinstance(node, dict) and node.get('videoId') + } + videos = OrderedDict() + for trailer_id in trailer_ids: + item = _supplemental_node_to_item(nodes_by_id.get(str(trailer_id), {})) + if item: + videos[item[0]] = item[1] + trailer_list = CustomVideoList({'videos': videos}) + trailer_list.is_supplemental_type = True + trailer_list.component_summary = {} + LOG.debug('Website GraphQL returned {} trailers for {}', len(videos), videoid) + return trailer_list + + def _req_video_list_supplemental_title_page(self, videoid): + try: + response = self.nfsession.session.get( + NETFLIX_TITLE_URL.format(videoid.value), + headers={ + 'Accept': 'text/html,application/xhtml+xml,application/xml', + 'User-Agent': common.get_user_agent(enable_android_mediaflag_fix=True) + }, + timeout=8) + response.raise_for_status() + react_context = website.extract_json(response.content, 'reactContext') + except (req_exceptions.RequestException, WebsiteParsingError) as exc: + LOG.warn('Title page trailer fallback failed for {} ({})', videoid, type(exc).__name__) + return self._empty_supplemental_list() + graphql_data = _title_page_graphql_data(response.content, react_context) + videos = _supplemental_videos_from_graphql_cache(graphql_data, videoid.value) + if not videos: + LOG.warn('No title page supplemental videos found for {}', videoid) + return self._empty_supplemental_list() + LOG.debug('Title page trailer fallback found {} supplemental videos for {}', len(videos), videoid) + trailer_list = CustomVideoList({'videos': videos}) + trailer_list.is_supplemental_type = True + trailer_list.component_summary = {} + return trailer_list + + @staticmethod + def _empty_supplemental_list(): + trailer_list = CustomVideoList({'videos': OrderedDict()}) + trailer_list.is_supplemental_type = True + trailer_list.component_summary = {} + return trailer_list @cache_utils.cache_output(cache_utils.CACHE_COMMON, identify_from_kwarg_name='chunked_video_list', ttl=900, ignore_self_class=True) @@ -258,21 +2128,148 @@ def req_video_list_chunked(self, chunked_video_list, perpetual_range_selector=No merged_response.update(perpetual_range_selector) return CustomVideoList(merged_response) - @cache_utils.cache_output(cache_utils.CACHE_SEARCH, identify_from_kwarg_name='search_term', - identify_append_from_kwarg_name='perpetual_range_start', ttl=900, ignore_self_class=True) def req_video_list_search(self, search_term, perpetual_range_start=None): """Retrieve a video list by search term""" LOG.debug('Requesting video list by search term "{}"', search_term) - base_path = ['search', 'byTerm', f'|{search_term}', 'titles', PATH_REQUEST_SIZE_STD] - paths = ([base_path + [['id', 'name', 'requestId', 'trackIds']]] + - build_paths(base_path + [RANGE_PLACEHOLDER, 'reference'], VIDEO_LIST_PARTIAL_PATHS)) - call_args = { - 'paths': paths, - 'length_params': ['searchlist', ['search', 'byReference']], - 'perpetual_range_start': perpetual_range_start - } - path_response = self.nfsession.perpetual_path_request(**call_args) - return SearchVideoList(path_response) + return self._req_video_list_search_graphql(search_term) + + def _req_video_list_search_graphql(self, search_term): + data = self._post_graphql( + 'SearchPageQueryResults', + _search_graphql_variables(search_term), + GRAPHQL_OP_SEARCH) + videos = OrderedDict() + path_response = {'videos': videos} + page = data.get('page') or {} + sections = (page.get('sections') or {}).get('edges') or [] + for section in sections: + section_node = section.get('node') or {} + if section_node.get('__typename') != 'PinotGallerySection': + continue + entities = (section_node.get('entities') or {}).get('edges') or [] + for entity_edge in entities: + item = _search_graphql_node_to_item(entity_edge.get('node') or {}) + if item: + video_id, video_data = item + videos.setdefault(video_id, video_data) + self._enrich_search_video_list(path_response) + LOG.debug('GraphQL search returned {} video results for "{}"', len(videos), search_term) + return CustomVideoList(path_response) + + def _enrich_search_video_list(self, path_response): + videos = path_response.get('videos') or {} + video_ids = list(videos)[:SEARCH_TITLE_PAGE_METADATA_LIMIT] + if not video_ids: + return + metadata_by_video = {video_id: {} for video_id in video_ids} + title_metadata_by_video = {} + max_workers = min(SEARCH_TITLE_PAGE_METADATA_WORKERS, len(video_ids)) + try: + metadata_request = self._prepare_metadata_request() + except Exception as exc: # pylint: disable=broad-except + LOG.debug('Search metadata request setup failed ({})', type(exc).__name__) + metadata_request = None + with ThreadPoolExecutor(max_workers=max_workers) as metadata_executor: + with ThreadPoolExecutor(max_workers=max_workers) as title_executor: + metadata_futures = ({ + metadata_executor.submit( + self._metadata_for_video_from_request, video_id, metadata_request): video_id + for video_id in video_ids + } if metadata_request else {}) + title_futures = { + title_executor.submit(_search_title_page_metadata, video_id): video_id + for video_id in video_ids + } + for future in as_completed(metadata_futures): + video_id = metadata_futures[future] + try: + metadata_by_video[video_id] = future.result() + except Exception as exc: # pylint: disable=broad-except + LOG.debug('Search metadata worker failed ({})', type(exc).__name__) + for future in as_completed(title_futures): + video_id = title_futures[future] + try: + title_metadata_by_video[video_id] = future.result() + except Exception as exc: # pylint: disable=broad-except + LOG.debug('Search title metadata worker failed ({})', type(exc).__name__) + for video_id in video_ids: + metadata = _merge_title_page_metadata( + metadata_by_video[video_id], title_metadata_by_video.get(video_id)) + video = _merge_search_metadata_video(videos[video_id], metadata) + videos[video_id] = video + normalize_metadata_references(path_response, video_id, metadata, video) + + def _enrich_search_title_page_metadata(self, metadata_by_video): + video_ids = [ + video_id + for video_id, metadata_video in metadata_by_video.items() + if not _metadata_has_reference_names(metadata_video) + ][:SEARCH_TITLE_PAGE_METADATA_LIMIT] + if not video_ids: + return + max_workers = min(SEARCH_TITLE_PAGE_METADATA_WORKERS, len(video_ids)) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_by_video_id = { + executor.submit(_search_title_page_metadata, video_id): video_id + for video_id in video_ids + } + for future in as_completed(future_by_video_id): + video_id = future_by_video_id[future] + try: + title_page_metadata = future.result() + except Exception as exc: # pylint: disable=broad-except + LOG.debug('Search title page metadata failed for {} ({})', video_id, type(exc).__name__) + continue + metadata_by_video[video_id] = _merge_title_page_metadata( + metadata_by_video.get(video_id), title_page_metadata) + + def _prepare_metadata_request(self): + # Resolve DB-backed auth/profile state before worker threads start. The add-on's + # shared SQLite connection and requests session are not safe for concurrent use. + endpoint_conf = ENDPOINTS['metadata'] + _, headers, params = self.nfsession._prepare_request_properties( # pylint: disable=protected-access + endpoint_conf, {'params': {}}) + request_headers = dict(self.nfsession.session.headers) + request_headers.update(headers) + # Stored Netflix cookies use PickleableCookieJar (a plain CookieJar), + # which has no .copy() method. Clone it into RequestsCookieJar so a + # missing method cannot disable all list/search metadata enrichment. + request_cookies = requests.cookies.merge_cookies( + requests.cookies.RequestsCookieJar(), self.nfsession.session.cookies) + api_url = G.LOCAL_DB.get_value('api_endpoint_url', table=TABLE_SESSION) + return SimpleNamespace( + url=f"{api_url.rstrip('/')}{endpoint_conf['address']}", + headers=request_headers, + params=params, + cookies=request_cookies) + + @staticmethod + def _metadata_for_video_from_request(video_id, metadata_request): + try: + params = dict(metadata_request.params) + params.update({'movieid': video_id, '_': int(time.time() * 1000)}) + response = requests.get( + metadata_request.url, + headers=metadata_request.headers, + params=params, + cookies=metadata_request.cookies, + timeout=(2, 4)) + response.raise_for_status() + metadata_data = response.json() if response.content else {} + return metadata_data.get('video') or {} + except (MetadataNotAvailable, KeyError, TypeError, ValueError, req_exceptions.RequestException): + LOG.debug('Search metadata enrichment skipped for video {}', video_id) + return {} + + def _metadata_for_video(self, video_id, context): + try: + metadata_data = self.nfsession.get_safe( + endpoint='metadata', + params={'movieid': video_id, '_': int(time.time() * 1000)}) + return metadata_with_title_page_fallback(video_id, metadata_data.get('video') or {}) + except (MetadataNotAvailable, KeyError, TypeError, req_exceptions.RequestException): + LOG.warn('{} metadata enrichment skipped for video {}', context, video_id) + return metadata_with_title_page_fallback(video_id) def req_subgenres(self, genre_id): """Retrieve sub-genres for the given genre""" @@ -287,6 +2284,9 @@ def req_datatype_video_list_full(self, context_name, switch_profiles=False): contains only minimal video info """ LOG.debug('Requesting the full video list for {}', context_name) + if context_name == 'mylist' and not switch_profiles: + return self._browser_mylist_video_list() + paths = (build_paths([context_name, 'az', RANGE_PLACEHOLDER], VIDEO_LIST_BASIC_PARTIAL_PATHS) + [[context_name, ['id', 'name', 'requestId', 'trackIds']]]) call_args = { @@ -312,22 +2312,31 @@ def req_datatype_video_list_byid(self, video_ids, custom_partial_paths=None): LOG.debug('Requesting a video list for {} videos', video_ids) paths = build_paths(['videos', video_ids], custom_partial_paths if custom_partial_paths else VIDEO_LIST_PARTIAL_PATHS) - path_response = self.nfsession.path_request(paths) - return CustomVideoList(path_response) + try: + path_response = self.nfsession.path_request(paths) + return CustomVideoList(path_response) + except req_exceptions.HTTPError as exc: + status_code = getattr(exc.response, 'status_code', None) + if status_code not in (404, 412): + raise + LOG.warn('Falling back to metadata video list for {} videos after pathEvaluator {}', + len(video_ids), status_code) + videos = OrderedDict() + for video_id in video_ids: + metadata_video = self._metadata_for_video(str(video_id), 'Video id list') + if metadata_video: + videos[str(video_id)] = _metadata_video_to_item(str(video_id), metadata_video) + return CustomVideoList({'videos': videos}) @cache_utils.cache_output(cache_utils.CACHE_COMMON, fixed_identifier='lolomo_category', identify_append_from_kwarg_name='category_name', ignore_self_class=True) def req_lolomo_category(self, category_name): """Retrieve LoLoMo by category lists""" LOG.debug('Requesting LoLoMo "{}" category lists', category_name) - paths = ([['lolomoByCategory', category_name, ['componentSummary']], - ['lolomoByCategory', category_name, {'from': 0, 'to': 10}, ['componentSummary']], - # Titles of first 4 videos in each video list (needed only to show titles in the plot description) - ['lolomoByCategory', category_name, - {'from': 0, 'to': 10}, {'from': 0, 'to': 3}, 'reference', ['title', 'summary']]] + - # Art for the first video of each context list (needed only to add art to the menu item) - build_paths(['lolomoByCategory', category_name, - {'from': 0, 'to': 10}, 0, 'reference'], ART_PARTIAL_PATHS)) - call_args = {'paths': paths} - path_response = self.nfsession.path_request(**call_args) - return LoLoMoCategory(path_response) + try: + return self._req_browser_lolomo_category(category_name) + except req_exceptions.HTTPError as exc: + if exc.response is None or exc.response.status_code not in (404, 412): + raise + LOG.warn('Falling back to current LoCo rows for LoLoMo category after pathEvaluator {}', exc.response.status_code) + return self._current_lolomo_category(category_name) diff --git a/resources/lib/services/nfsession/msl/profiles.py b/resources/lib/services/nfsession/msl/profiles.py index ebf94e0ac..f30776428 100644 --- a/resources/lib/services/nfsession/msl/profiles.py +++ b/resources/lib/services/nfsession/msl/profiles.py @@ -81,7 +81,7 @@ def _profile_strings(base, tails): def enabled_profiles(): """Return a list of all base and enabled additional profiles""" return (PROFILES['base'] + - PROFILES['h264'] + PROFILES['h264_prk_qc'] + + PROFILES['h264'] + _subtitle_profiles() + _additional_profiles('vp9profile0', 'enable_vp9_profiles') + _additional_profiles('vp9profile2', ['enable_vp9_profiles', 'enable_vp9.2_profiles']) + diff --git a/resources/lib/services/nfsession/nfsession_ops.py b/resources/lib/services/nfsession/nfsession_ops.py index 41a86982d..b9376459f 100644 --- a/resources/lib/services/nfsession/nfsession_ops.py +++ b/resources/lib/services/nfsession/nfsession_ops.py @@ -10,6 +10,8 @@ import time from datetime import datetime, timedelta +import requests +import requests.exceptions as req_exceptions import xbmc import resources.lib.common as common @@ -17,16 +19,42 @@ from resources.lib.common import cache_utils from resources.lib.common.exceptions import (NotLoggedInError, MissingCredentialsError, WebsiteParsingError, MbrStatusAnonymousError, MetadataNotAvailable, LoginValidateError, - HttpError401, InvalidProfilesError, ErrorMsgNoReport) + InvalidProfilesError, ErrorMsgNoReport, CacheMiss) from resources.lib.globals import G from resources.lib.kodi import ui +from resources.lib.services.nfsession.directorybuilder.dir_path_requests import (_metadata_image_url, + _metadata_trailer_url, + metadata_with_title_page_fallback, + normalize_metadata_references) from resources.lib.services.nfsession.session.path_requests import SessionPathRequests from resources.lib.utils import cookies from resources.lib.utils.api_paths import (EPISODES_PARTIAL_PATHS, ART_PARTIAL_PATHS, build_paths, - VIDEO_LIST_PARTIAL_PATHS) + VIDEO_LIST_PARTIAL_PATHS, ART_SIZE_FHD, ART_SIZE_POSTER) from resources.lib.utils.logging import LOG, measure_exec_time_decorator +def _is_playable_direct_trailer_url(trailer_url): + """Require the direct fallback to resolve to actual video content.""" + if not isinstance(trailer_url, str) or not trailer_url.startswith('http'): + return False + try: + response = requests.get( + trailer_url, + headers={ + 'Range': 'bytes=0-0', + 'User-Agent': common.get_user_agent(enable_android_mediaflag_fix=True) + }, + stream=True, + timeout=(2, 4)) + try: + content_type = (response.headers.get('content-type') or '').lower() + return response.status_code in (200, 206) and content_type.startswith('video/') + finally: + response.close() + except req_exceptions.RequestException: + return False + + class NFSessionOperations(SessionPathRequests): """Provides methods to perform operations within the Netflix session""" @@ -36,6 +64,7 @@ def __init__(self): self.slots = [ self.get_safe, self.post_safe, + self.post_graphql, self.login, self.login_auth_data, self.logout, @@ -47,7 +76,8 @@ def __init__(self): self.activate_profile, self.parental_control_data, self.get_metadata, - self.get_videoid_info + self.get_videoid_info, + self.get_direct_trailer ] # Share the activate profile function to SessionBase class self.external_func_activate_profile = self.activate_profile @@ -96,8 +126,7 @@ def refresh_session_data(self, update_profiles): def activate_profile(self, guid): """Set the profile identified by guid as active""" LOG.debug('Switching to profile {}', guid) - current_active_guid = G.LOCAL_DB.get_active_profile_guid() - if guid == current_active_guid: + if guid == G.LOCAL_DB.get_active_profile_guid(): LOG.info('The profile guid {} is already set, activation not needed.', guid) return if xbmc.Player().isPlayingVideo(): @@ -105,29 +134,14 @@ def activate_profile(self, guid): # (MSL/NFSession) causing a failure in the HTTP request or sending data on the wrong profile raise ErrorMsgNoReport('It is not possible select a profile while a video is playing.') LOG.info('Activating profile {}', guid) - - # INIT Method 1 - HTTP mode - response = self.get_safe('switch_profile', params={'tkn': guid}) - self.auth_url = self.website_extract_session_data(response)['auth_url'] - # END Method 1 - - # INIT Method 2 - API mode **** 07/2026 not working anymore **** - # try: - # timestamp = time.time() - # response = self.get_safe(endpoint='activate_profile', - # params={'switchProfileGuid': guid, - # '_': int(timestamp * 1000), - # 'authURL': self.auth_url}) - # if response.get('status') != 'success': - # raise InvalidProfilesError('Unable to access to the selected profile.') - # except HttpError401 as exc: - # # Profile guid not more valid - # raise InvalidProfilesError('Unable to access to the selected profile.') from exc - # Retrieve browse page to update authURL - # response = self.get_safe('browse') - # self.auth_url = website.extract_session_data(response)['auth_url'] - # END Method 2 - + try: + # Use /SwitchProfile endpoint to switch the active profile server-side + self.get_safe('switch_profile', params={'tkn': guid}) + # Fetch browse page to get a fresh authURL for the new profile + response = self.get_safe('browse') + self.auth_url = website.extract_session_data(response)['auth_url'] + except Exception as exc: + raise InvalidProfilesError('Unable to access to the selected profile.') from exc G.LOCAL_DB.switch_active_profile(guid) G.CACHE_MANAGEMENT.identifier_prefix = guid cookies.save(self.session.cookies) @@ -199,6 +213,42 @@ def get_metadata(self, videoid, refresh=False): metadata_data = self._metadata(video_id=parent_videoid), None return metadata_data + def get_direct_trailer(self, videoid): + """Return a fresh, verified public trailer fallback for a title.""" + cache_identifier = f'direct_trailer_{videoid}' + try: + return G.CACHE.get(cache_utils.CACHE_SUPPLEMENTAL, cache_identifier) + except CacheMiss: + pass + try: + metadata_data = self.get_safe( + endpoint='metadata', + params={'movieid': videoid.value, '_': int(time.time() * 1000)}) + metadata_video = metadata_with_title_page_fallback( + videoid.value, metadata_data.get('video') or {}) + except (MetadataNotAvailable, AttributeError, KeyError, TypeError, req_exceptions.RequestException): + result = {} + G.CACHE.add(cache_utils.CACHE_SUPPLEMENTAL, cache_identifier, result) + return result + trailer_url = _metadata_trailer_url(metadata_video) + if not _is_playable_direct_trailer_url(trailer_url): + result = {} + G.CACHE.add(cache_utils.CACHE_SUPPLEMENTAL, cache_identifier, result) + return result + trailer_data = metadata_video.get('trailer') or {} + trailer_title = trailer_data.get('name') if isinstance(trailer_data, dict) else '' + poster = _metadata_image_url( + metadata_video, ('boxart', 'boxArt', 'boxarts'), portrait=True) + result = { + 'url': trailer_url, + 'title': trailer_title or metadata_video.get('title') or '', + 'synopsis': metadata_video.get('synopsis') or metadata_video.get('regularSynopsis') or '', + 'year': metadata_video.get('year') or metadata_video.get('releaseYear') or 0, + 'poster': poster + } + G.CACHE.add(cache_utils.CACHE_SUPPLEMENTAL, cache_identifier, result) + return result + def _episode_metadata(self, episode_videoid, tvshow_videoid, refresh_cache=False): if refresh_cache: G.CACHE.delete(cache_utils.CACHE_METADATA, str(tvshow_videoid)) @@ -261,17 +311,106 @@ def get_videoid_info(self, videoid): try: infos = get_info(videoid, None, None, profile_language_code)[0] art = get_art(videoid, None, profile_language_code) + if infos.get('Cast') and (videoid.mediatype == common.VideoId.EPISODE or infos.get('Trailer')): + return infos, art + LOG.debug('Cached video info for {} is missing cast/trailer; refreshing metadata', videoid) except (AttributeError, TypeError): - if videoid.mediatype == common.VideoId.EPISODE: - paths = (build_paths(['videos', int(videoid.value)], EPISODES_PARTIAL_PATHS) + - build_paths(['videos', int(videoid.tvshowid)], ART_PARTIAL_PATHS + [[['title', 'delivery']]])) - else: - paths = build_paths(['videos', int(videoid.value)], VIDEO_LIST_PARTIAL_PATHS) + pass + if videoid.mediatype == common.VideoId.EPISODE: + paths = (build_paths(['videos', int(videoid.value)], EPISODES_PARTIAL_PATHS) + + build_paths(['videos', int(videoid.tvshowid)], + ART_PARTIAL_PATHS + [[['title', 'delivery']]])) + else: + paths = build_paths(['videos', int(videoid.value)], VIDEO_LIST_PARTIAL_PATHS) + try: raw_data = self.path_request(paths) + except req_exceptions.HTTPError as exc: + LOG.warn('Video info pathEvaluator lookup failed: {}. Falling back to metadata endpoint.', exc) + raw_data = self._get_videoid_info_metadata(videoid) + infos = get_info(videoid, raw_data['videos'][videoid.value], raw_data, profile_language_code)[0] + if (videoid.mediatype != common.VideoId.EPISODE and + (not infos.get('Cast') or not infos.get('Trailer'))): + LOG.debug('Video info for {} is missing cast/trailer; refreshing from metadata endpoint', videoid) + raw_data = self._get_videoid_info_metadata(videoid) infos = get_info(videoid, raw_data['videos'][videoid.value], raw_data, profile_language_code)[0] - art = get_art(videoid, raw_data['videos'][videoid.value], profile_language_code) + art = get_art(videoid, raw_data['videos'][videoid.value], profile_language_code) return infos, art + def _get_videoid_info_metadata(self, videoid): + metadata_data = self.get_safe( + endpoint='metadata', + params={'movieid': videoid.value, '_': int(time.time() * 1000)}) + video = metadata_with_title_page_fallback(videoid.value, metadata_data['video']) + item = self._metadata_video_to_path_item(videoid, video) + videos = {videoid.value: item} + raw_data = {'videos': videos} + normalize_metadata_references(raw_data, videoid.value, video, item) + if videoid.mediatype == common.VideoId.EPISODE and videoid.tvshowid: + videos.setdefault(videoid.tvshowid, { + 'title': {'value': video.get('seriesTitle') or video.get('showTitle') or ''}, + 'delivery': {'value': {}} + }) + return raw_data + + @staticmethod + def _metadata_video_to_path_item(videoid, video): + title = video.get('title') or str(videoid.value) + synopsis = video.get('synopsis') or video.get('regularSynopsis') or '' + boxart_url = NFSessionOperations._find_metadata_image_url(video, ('boxArt', 'boxart', 'artwork')) + still_url = NFSessionOperations._find_metadata_image_url(video, ('interestingMoment', 'interestingMomentUrl')) + item = { + 'summary': {'value': { + 'id': int(videoid.value), + 'type': videoid.mediatype, + 'name': title + }}, + 'title': {'value': title}, + 'synopsis': {'value': synopsis}, + 'regularSynopsis': {'value': synopsis}, + 'runtime': {'value': video.get('runtime') or 0}, + 'releaseYear': {'value': video.get('year') or video.get('releaseYear') or 0}, + 'delivery': {'value': video.get('delivery') or {}}, + 'availability': {'value': {'isPlayable': True}}, + 'queue': {'value': {'inQueue': False}}, + 'inRemindMeList': {'value': False}, + 'bookmarkPosition': {'value': (video.get('bookmark') or {}).get('offset', 0)}, + 'creditsOffset': {'value': video.get('creditsOffset') or 0}, + 'watchedToEndOffset': {'value': video.get('watchedToEndOffset') or 0}, + 'watched': {'value': bool((video.get('bookmark') or {}).get('watchedDate'))}, + 'trackIds': {'value': {}}, + 'requestId': {'value': ''} + } + if boxart_url: + art_value = {'url': boxart_url} + item['boxarts'] = {ART_SIZE_POSTER: {'jpg': {'value': art_value}}} + item['itemSummary'] = {'value': {'id': int(videoid.value), 'title': title, 'boxArt': {'url': boxart_url}}} + if still_url: + item['interestingMoment'] = {ART_SIZE_FHD: {'jpg': {'value': {'url': still_url}}}} + return item + + @staticmethod + def _find_metadata_image_url(video, keys): + for key in keys: + value = video.get(key) + if isinstance(value, str) and value.startswith('http'): + return value + if isinstance(value, dict): + url = NFSessionOperations._find_url_in_dict(value) + if url: + return url + return '' + + @staticmethod + def _find_url_in_dict(data): + for value in data.values(): + if isinstance(value, str) and value.startswith('http'): + return value + if isinstance(value, dict): + url = NFSessionOperations._find_url_in_dict(value) + if url: + return url + return '' + def get_loco_data(self): """ Get the LoCo root id and the continueWatching list data references diff --git a/resources/lib/services/nfsession/session/access.py b/resources/lib/services/nfsession/session/access.py index fe44cb0ef..3507b4ed4 100644 --- a/resources/lib/services/nfsession/session/access.py +++ b/resources/lib/services/nfsession/session/access.py @@ -8,12 +8,15 @@ SPDX-License-Identifier: MIT See LICENSES/MIT.md for more information. """ +import json import re +from urllib.parse import quote import resources.lib.utils.website as website import resources.lib.common as common import resources.lib.utils.cookies as cookies import resources.lib.kodi.ui as ui +import resources.lib.services.nfsession.session.endpoints as ep from resources.lib.common.exceptions import (LoginValidateError, NotConnected, NotLoggedInError, MbrStatusNeverMemberError, MbrStatusFormerMemberError, LoginError, MissingCredentialsError, MbrStatusAnonymousError, WebsiteParsingError) @@ -23,6 +26,59 @@ from resources.lib.services.nfsession.session.http_requests import SessionHTTPRequests from resources.lib.utils.logging import LOG, measure_exec_time_decorator +CLCS_GRAPHQL_URL = ep.BASE_URL + '/graphql' +CLCS_SCREEN_UPDATE_ID = '0ed5cd22-de4e-4883-bf7a-ed255ab88664' +CLCS_QUERY_VERSION = 102 +# Time waited by the website for the reCAPTCHA script before submitting the sign in with the error +RECAPTCHA_TIMEOUT_MS = 2730 + +# The website serves the CLCS login flow only to recent browsers, +# with an outdated user agent it falls back to a static form protected by reCAPTCHA +BROWSER_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0' +BROWSER_HEADERS = { + 'User-Agent': BROWSER_USER_AGENT, + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + 'Sec-GPC': '1' +} + + +# Cipher list used by Firefox, the default one of python is recognizable as a non browser client +FIREFOX_CIPHERS = ( + 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:' + 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:' + 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:' + 'ECDHE-ECDSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA:' + 'ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:' + 'AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA:AES256-SHA' +) + + +def _mount_browser_tls(session): + """Use the TLS settings of a browser for the login requests""" + try: + import ssl + from requests.adapters import HTTPAdapter + from urllib3.poolmanager import PoolManager + + class _BrowserTLSAdapter(HTTPAdapter): + def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs): + context = ssl.create_default_context() + context.set_ciphers(FIREFOX_CIPHERS) + context.options |= getattr(ssl, 'OP_NO_COMPRESSION', 0) + pool_kwargs['ssl_context'] = context + self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, + block=block, **pool_kwargs) + + session.mount('https://www.netflix.com', _BrowserTLSAdapter()) + except Exception as exc: # pylint: disable=broad-except + LOG.warn('Cannot set the browser TLS settings ({})', exc) + class SessionAccess(SessionCookie, SessionHTTPRequests): """Handle the authentication access""" @@ -140,7 +196,14 @@ def login_auth_data(self, data=None, password=None): if exc.response.status_code == 500: # This endpoint raise HTTP error 500 when the password is wrong raise LoginError(common.get_local_string(12344)) from exc - raise + if exc.response.status_code not in (404, 410): + raise + # The legacy profilehub endpoint has been removed for some accounts. + # The Auth Key cookies were already validated above, so allow login to + # continue while retaining the supplied password for MSL compatibility. + LOG.warn('Password verification endpoint is unavailable (HTTP {}); ' + 'continuing Auth Key login without password verification', + exc.response.status_code) common.set_credentials({'email': email, 'password': password}) LOG.info('Login successful') ui.show_notification(common.get_local_string(30109)) @@ -149,19 +212,31 @@ def login_auth_data(self, data=None, password=None): @measure_exec_time_decorator(is_immediate=True) def login(self, credentials=None): - """Perform account login with credentials""" + """Perform account login with credentials by driving the website CLCS login flow""" try: - # First we get the authentication url without logging in, required for login API call self.session.cookies.clear() - react_context = website.extract_json(self.get('login'), 'reactContext') - auth_url = website.extract_api_data(react_context)['auth_url'] + credentials = credentials or common.get_credentials() LOG.debug('Logging in with credentials') - login_response = self.post( - 'login', - headers={'Accept-Language': _get_accept_language_string(react_context)}, - data=_login_payload(credentials or common.get_credentials(), auth_url, react_context)) + _mount_browser_tls(self.session) + page = self._get_login_page() + server_state, server_screen_update, country, action_fields, app_version = _extract_clcs_bootstrap(page) + # Send the same fields of the website, the password is not sent to let the website + # choose the verification method for the account (e.g. the one-time code by e-mail) + input_fields = [ + _clcs_field('userLoginId', 'stringValue', credentials['email']), + _clcs_field('countryCode', 'stringValue', country['code']), + _clcs_field('countryIsoCode', 'stringValue', country['iso']) + ] + input_fields += _clcs_recaptcha_fields(action_fields) + self._clcs_app_version = app_version + # The website submits the sign in after having waited the reCAPTCHA script, + # send it with the same delay that is declared to the website + import time + time.sleep(RECAPTCHA_TIMEOUT_MS / 1000) + result = self._clcs_screen_update(server_state, server_screen_update, input_fields) + self._clcs_complete_login(result, credentials, country) - website.extract_session_data(login_response, validate=True, update_profiles=True) + website.extract_session_data(self.get('browse'), validate=True, update_profiles=True) if credentials: # Save credentials only when login has succeeded common.set_credentials(credentials) @@ -183,6 +258,67 @@ def login(self, credentials=None): LOG.error(traceback.format_exc()) raise + def _get_login_page(self): + """Request the login page with modern browser headers, needed to get the CLCS login flow""" + response = self.session.get(url=ep.BASE_URL + '/login', headers=BROWSER_HEADERS, timeout=8) + response.raise_for_status() + return response.text + + def _clcs_screen_update(self, server_state, server_screen_update, input_fields): + """Send a single step of the CLCS login flow to the website GraphQL gateway""" + payload = { + 'operationName': 'CLCSScreenUpdate', + 'variables': { + 'format': 'HTML', + 'imageFormat': 'PNG', + 'locale': 'en-US', + 'serverState': server_state, + 'serverScreenUpdate': server_screen_update, + 'inputFields': input_fields + }, + 'extensions': {'persistedQuery': {'id': CLCS_SCREEN_UPDATE_ID, 'version': CLCS_QUERY_VERSION}} + } + # Use separators with dumps because Netflix rejects spaces + data = json.dumps(payload, separators=(',', ':')) + response = self.session.post( + url=CLCS_GRAPHQL_URL, + data=data.encode('utf-8'), + headers=_graphql_login_headers(server_state, getattr(self, '_clcs_app_version', '')), + timeout=8) + response.raise_for_status() + decoded = response.json() if response.content else {} + if decoded.get('errors'): + raise LoginError(decoded['errors'][0].get('message', 'GraphQL error')) + return decoded + + def _clcs_complete_login(self, result, credentials, country): + """Walk the CLCS screens (password / one-time code) until the login succeeds""" + password_submitted = False + for _ in range(6): + data = (result or {}).get('data', {}).get('result', {}) + typename = data.get('__typename') + if typename == 'CLCSScreenUpdateEffect': + if data.get('status') == 'SUCCESS': + return + raise LoginError(_find_clcs_message(data) or common.get_local_string(30008)) + if typename != 'CLCSScreenUpdateTransition': + raise LoginError(_find_clcs_message(data) or common.get_local_string(30008)) + screen = data.get('screen') or {} + server_state = screen.get('serverState') + action, kind = _select_clcs_action(screen) + if not action: + _log_clcs_screen_diagnostics(screen) + raise LoginError(_find_clcs_message(screen) or common.get_local_string(30008)) + if kind == 'password': + if password_submitted: + # The password screen is asked again, the credentials have been refused + _log_clcs_screen_diagnostics(screen) + raise LoginError(_find_clcs_message(screen) or common.get_local_string(30008)) + password_submitted = True + input_fields = _build_clcs_fields(action['inputFieldRequirements'], credentials, kind, screen, country) + result = self._clcs_screen_update(server_state, action['serverScreenUpdate'], input_fields) + raise LoginError(common.get_local_string(30008)) + @measure_exec_time_decorator(is_immediate=True) def logout(self): """Logout of the current account and reset the session""" @@ -227,46 +363,235 @@ def logout(self): common.container_update(G.BASE_URL, True) -def _login_payload(credentials, auth_url, react_context): - country_id = react_context['models']['signupContext']['data']['geo']['requestCountry']['id'] - country_codes = react_context['models']['countryCodes']['data']['codes'] - try: - country_code = '+' + next(dict_item for dict_item in country_codes if dict_item["id"] == country_id)['code'] - except StopIteration: - country_code = '' - # 25/08/2020 since a few days there are login problems, by returning the "incorrect password" error even - # when it is correct, it seems that setting 'rememberMe' to 'false' increases a bit the probabilities of success - return { - 'userLoginId': credentials.get('email'), - 'password': credentials.get('password'), - 'rememberMe': 'false', - 'flow': 'websiteSignUp', - 'mode': 'login', - 'action': 'loginAction', - 'withFields': 'rememberMe,nextPage,userLoginId,password,countryCode,countryIsoCode', - 'authURL': auth_url, - 'nextPage': '', - 'showPassword': '', - 'countryCode': country_code, - 'countryIsoCode': country_id +def _extract_clcs_bootstrap(content): + """Read the initial CLCS screen state embedded in the login page""" + html = content.decode('utf-8') if isinstance(content, bytes) else content + server_state = _find_page_value(html, 'serverState') + # Get the sign in action, that provides the screen update value and the fields to be sent + actions = re.findall(r'"inputFieldRequirements":\[(.*?)\],"preload":[^,]*,"serverScreenUpdate":"([^"]+)"', + html, re.DOTALL) + screen_updates = [website.decode_javascript_string(value) for _, value in actions] + if not screen_updates: + screen_updates = [website.decode_javascript_string(value) + for value in re.findall(r'"serverScreenUpdate":"([^"]+)"', html)] + if not server_state or not screen_updates: + _log_login_page_diagnostics(html) + raise WebsiteParsingError('CLCS login state not found in the login page') + action_fields = re.findall(r'"id":"(\w+)"', actions[-1][0]) if actions else [] + build_match = re.search(r'"BUILD_IDENTIFIER":"([^"]+)"', html) + app_version = build_match.group(1) if build_match else '' + LOG.debug('CLCS login, sign in action fields {}', action_fields) + iso_match = re.search(r'"requestCountry":\{[^{}]*?"id":"([A-Z]{2})"', html) + country = {'iso': iso_match.group(1) if iso_match else 'US', 'code': '1'} + return server_state, screen_updates[-1], country, action_fields, app_version + + +def _graphql_login_headers(server_state=None, app_version=''): + """Set the same headers of the website, needed to get the request accepted""" + import uuid + page_url = ep.BASE_URL + '/login' + if server_state: + page_url += '?serverState=' + quote(server_state, safe='') + headers = { + 'Accept': '*/*', + 'Content-Type': 'application/json', + 'Origin': ep.BASE_URL, + 'Referer': page_url, + 'x-netflix.request.originating.url': page_url, + 'x-netflix.request.id': uuid.uuid4().hex, + 'x-netflix.request.toplevel.uuid': str(uuid.uuid4()), + 'x-netflix.request.attempt': '1', + 'x-netflix.request.clcs.bucket': 'high', + 'x-netflix.request.client.context': '{"appView":"identification","action":"Submitted","appstate":"foreground"}', + 'x-netflix.context.ui-flavor': 'akira', + 'x-netflix.context.operation-name': 'CLCSScreenUpdate', + 'x-netflix.context.locales': 'en-us', + 'x-netflix.context.hawkins-version': '5.26.0', + 'x-netflix.context.app-version': app_version, + 'Sec-Fetch-Dest': 'empty', + 'Sec-Fetch-Mode': 'cors', + 'Sec-Fetch-Site': 'same-origin' } + headers.update({key: value for key, value in BROWSER_HEADERS.items() + if key in ('User-Agent', 'Accept-Language', 'Sec-GPC')}) + return headers -def _get_accept_language_string(react_context): - # pylint: disable=consider-using-f-string - # Set the HTTP header 'Accept-Language' allow to get http strings in the right language, - # and also influence the reactContext data (locale data and messages strings). - # Locale is usually automatically determined by the browser, - # we try get the locale code by reading the locale set as default in the reactContext. - supported_locales = react_context['models']['signupContext']['data']['geo']['supportedLocales'] - try: - locale = next(dict_item for dict_item in supported_locales if dict_item["default"] is True)['locale'] - except StopIteration: - locale = '' - locale_fallback = 'en-US' - if locale and locale != locale_fallback: - return '{loc},{loc_l};q=0.9,{loc_fb};q=0.8,{loc_fb_l};q=0.7'.format( - loc=locale, loc_l=locale[:2], - loc_fb=locale_fallback, loc_fb_l=locale_fallback[:2]) - return '{loc},{loc_l};q=0.9'.format( - loc=locale_fallback, loc_l=locale_fallback[:2]) +def _find_page_value(html, key): + match = re.search(r'"' + key + r'":"([^"]*)"', html) + return website.decode_javascript_string(match.group(1)) if match else None + + +def _log_login_page_diagnostics(html): + """Dump the login form so the form-post login can be rebuilt from real data""" + LOG.error('LOGIN diagnostics: page length {}', len(html)) + for form in re.findall(r']*>', html): + LOG.error('LOGIN diagnostics: form {}', form) + for input_tag in re.findall(r']*>', html): + LOG.error('LOGIN diagnostics: input {}', input_tag) + for assignment in re.findall(r"nonmemberStaticFramework[^\n;]{0,120}=\s*[^\n;]{0,500};", html): + LOG.error('LOGIN diagnostics: data {}', assignment.strip()) + for marker in ['authURL', 'nonmemberStaticFramework.data', 'recaptcha', 'useEnterprise', 'action']: + index = html.find(marker) + if index != -1: + LOG.error('LOGIN diagnostics: context [{}] {}', marker, html[max(0, index - 80):index + 320]) + + +def _clcs_field(name, value_type, value): + return {'name': name, 'value': {value_type: value}} + + +def _iter_clcs_nodes(node): + if isinstance(node, dict): + yield node + for value in node.values(): + yield from _iter_clcs_nodes(value) + elif isinstance(node, list): + for value in node: + yield from _iter_clcs_nodes(value) + + +def _select_clcs_action(screen): + """Pick the screen action that asks for a one-time code or the password""" + otp_action = None + password_action = None + for node in _iter_clcs_nodes(screen): + if node.get('__typename') != 'CLCSRequestScreenUpdate': + continue + requirements = node.get('inputFieldRequirements') + if not requirements or not node.get('serverScreenUpdate'): + continue + field_ids = [(req.get('field') or {}).get('id', '') for req in requirements] + kind = _classify_clcs_fields(field_ids) + if kind == 'otp' and not otp_action: + otp_action = node + elif kind == 'password' and not password_action: + password_action = node + if otp_action: + return otp_action, 'otp' + if password_action: + return password_action, 'password' + return None, None + + +def _classify_clcs_fields(field_ids): + lowered = [field_id.lower() for field_id in field_ids] + if any('otp' in fid or 'pin' in fid or 'challenge' in fid for fid in lowered): + return 'otp' + if any('password' in fid or 'passcode' in fid or fid == 'credential' for fid in lowered): + return 'password' + return None + + +def _build_clcs_fields(requirements, credentials, kind, screen, country): + input_fields = [] + for requirement in requirements: + field = requirement.get('field') or {} + field_id = field.get('id') + if not field_id: + continue + lowered = field_id.lower() + if 'otp' in lowered or 'pin' in lowered or 'challenge' in lowered: + code = ui.ask_for_input(_find_clcs_title(screen) or 'Enter the code Netflix sent you') + if not code: + raise MissingCredentialsError + value = code.strip() + elif 'password' in lowered or 'passcode' in lowered or field_id == 'credential': + value = credentials['password'] + elif field_id == 'userLoginId': + value = credentials['email'] + elif field_id == 'countryCode': + value = country['code'] + elif field_id == 'countryIsoCode': + value = country['iso'] + elif field_id == 'rememberMe': + value = True + else: + # The reCaptcha fields are optional, the other fields keep their initial value + if not lowered.startswith('recaptcha'): + LOG.warn('CLCS login, unhandled required field {} ({})', field_id, field.get('fieldType')) + value = field.get('initialStringValue') or '' + input_fields.append(_clcs_typed_field(field, field_id, value)) + if kind == 'password' and not any(fld['name'] == 'password' for fld in input_fields): + input_fields.append(_clcs_field('password', 'stringValue', credentials['password'])) + return input_fields + + +def _clcs_recaptcha_fields(field_ids): + """The reCaptcha token can be produced only by the Google script in a web browser, + report the same timeout error of the website when the script cannot be executed, + the token must not be sent otherwise the request is refused""" + input_fields = [] + if 'recaptchaError' in field_ids: + input_fields.append(_clcs_field('recaptchaError', 'stringValue', 'RESPONSE_TIMED_OUT')) + if 'recaptchaResponseTime' in field_ids: + input_fields.append(_clcs_field('recaptchaResponseTime', 'intValue', RECAPTCHA_TIMEOUT_MS)) + return input_fields + + +def _clcs_typed_field(field, field_id, value): + """Build an input field with the value type declared by the screen""" + field_type = field.get('fieldType') or '' + if 'Boolean' in field_type: + return _clcs_field(field_id, 'boolValue', bool(value)) + if 'Integer' in field_type or 'Number' in field_type: + try: + return _clcs_field(field_id, 'intValue', int(value)) + except (TypeError, ValueError): + return _clcs_field(field_id, 'intValue', 0) + return _clcs_field(field_id, 'stringValue', '' if value is None else str(value)) + + +def _find_clcs_title(screen): + for node in _iter_clcs_nodes(screen): + if node.get('__typename') == 'CLCSText' and node.get('testId') == 'title': + return (node.get('plainContent') or {}).get('value') + return None + + +def _find_clcs_message(node): + """Get the message shown by the screen, the alerts of the actions are a generic fallback text""" + for item in _iter_clcs_nodes(node): + if item.get('__typename') != 'CLCSText': + continue + test_id = (item.get('testId') or '').lower() + value = (item.get('plainContent') or {}).get('value') + if value and ('error' in test_id or 'message' in test_id): + return value + return _find_clcs_title(node) + + +def _log_clcs_screen_diagnostics(screen): + """Dump the received screen to understand which step the login flow is asking for""" + LOG.debug('CLCS screen: title {}', _find_clcs_title(screen)) + texts = [] + actions = [] + for node in _iter_clcs_nodes(screen): + typename = node.get('__typename') + if typename == 'CLCSText': + value = (node.get('plainContent') or {}).get('value') + if value: + texts.append(f"[{node.get('testId')}] {value}") + elif typename == 'CLCSRequestScreenUpdate' and node.get('inputFieldRequirements'): + field_ids = [(req.get('field') or {}).get('id') for req in node['inputFieldRequirements']] + actions.append(field_ids) + for text in texts[:25]: + LOG.debug('CLCS screen text: {}', text) + # The messages are not always in a text component, get every localized string of the screen + strings = [] + for node in _iter_clcs_nodes(screen): + if node.get('__typename') in ('GrowthLocalizedString', 'GrowthLocalizedFormattedString'): + value = node.get('value') + if value and value not in strings: + strings.append(value) + for value in strings[:30]: + LOG.debug('CLCS screen string: {}', value) + for node in _iter_clcs_nodes(screen): + test_id = node.get('testId') or '' + if any(key in test_id for key in ('alert', 'error', 'message')): + LOG.debug('CLCS screen alert node [{}]: {}', test_id, json.dumps(node)[:1200]) + for field_ids in actions: + LOG.debug('CLCS screen action fields: {}', field_ids) + components = sorted({node.get('__typename') for node in _iter_clcs_nodes(screen) + if isinstance(node.get('__typename'), str)}) + LOG.debug('CLCS screen components: {}', components) diff --git a/resources/lib/services/nfsession/session/endpoints.py b/resources/lib/services/nfsession/session/endpoints.py index ee6485406..99223fa4b 100644 --- a/resources/lib/services/nfsession/session/endpoints.py +++ b/resources/lib/services/nfsession/session/endpoints.py @@ -77,13 +77,6 @@ 'use_default_params': False, 'add_auth_url': None, 'accept': '*/*'}, - # **** 07/2026 not working anymore **** - # 'activate_profile': - # {'address': '/api/shakti/mre/profiles/switch', - # 'is_api_call': False, - # 'use_default_params': False, - # 'add_auth_url': None, - # 'accept': '*/*'}, 'profile_lock': {'address': '/api/shakti/mre/profileLock', 'is_api_call': False, diff --git a/resources/lib/services/nfsession/session/http_requests.py b/resources/lib/services/nfsession/session/http_requests.py index cff94bf74..2e453908d 100644 --- a/resources/lib/services/nfsession/session/http_requests.py +++ b/resources/lib/services/nfsession/session/http_requests.py @@ -23,9 +23,13 @@ from resources.lib.services.nfsession.session.base import SessionBase from resources.lib.services.nfsession.session.endpoints import ENDPOINTS, BASE_URL from resources.lib.utils import cookies +from resources.lib.utils.esn import get_website_esn from resources.lib.utils.logging import LOG, measure_exec_time_decorator +GRAPHQL_URL = 'https://web.prod.cloud.netflix.com/graphql' + + class SessionHTTPRequests(SessionBase): """Manages the HTTP requests""" @@ -43,6 +47,30 @@ def post(self, endpoint, **kwargs): endpoint=endpoint, **kwargs) + @measure_exec_time_decorator(is_immediate=True) + def post_graphql(self, operation_name, variables, operation_id, referer=None): + """Execute a persisted GraphQL request against the website GraphQL gateway.""" + self.assert_logged_in() + payload = { + 'operationName': operation_name, + 'variables': variables, + 'extensions': {'persistedQuery': {'id': operation_id, 'version': 102}} + } + LOG.debug('Executing GraphQL request: {}', operation_name) + start = time.perf_counter() + response = self.session.post( + url=GRAPHQL_URL, + json=payload, + headers=_graphql_headers(referer), + timeout=8) + LOG.debug('Request took {}s', time.perf_counter() - start) + LOG.debug('Request returned status code {}', response.status_code) + response.raise_for_status() + decoded_response = response.json() if response.content else {} + if decoded_response.get('errors'): + raise APIError(decoded_response['errors'][0].get('message')) + return decoded_response + @measure_exec_time_decorator(is_immediate=True) def _request_call(self, method, endpoint, **kwargs): return self._request(method, endpoint, None, **kwargs) @@ -161,6 +189,8 @@ def _prepare_request_properties(self, endpoint_conf, kwargs): headers['x-netflix.request.client.user.guid'] = G.LOCAL_DB.get_active_profile_guid() if endpoint_conf.get('content_type'): headers['Content-Type'] = endpoint_conf['content_type'] + if endpoint_conf['address'] == '/pathEvaluator': + _add_path_evaluator_headers(headers) headers.update(custom_headers) # If needed override headers # Meanings parameters known: # drmSystem DRM used @@ -170,10 +200,15 @@ def _prepare_request_properties(self, endpoint_conf, kwargs): # it is still added in an 'empty' form in the response if endpoint_conf['use_default_params']: params = { + 'webp': 'false', 'drmSystem': 'widevine', + 'isVolatileBillboardsEnabled': 'true', + 'isTop10Supported': 'true', + 'hasVideoMerchInBob': 'false', + 'hasVideoMerchInJaw': 'false', 'falcor_server': '0.1.0', - 'withSize': 'false', - 'materialize': 'false', + 'withSize': 'true', + 'materialize': 'true', 'original_path': '/shakti/mre/pathEvaluator' } if endpoint_conf['add_auth_url'] == 'to_params': @@ -208,6 +243,70 @@ def _api_url(endpoint_address): return f'{baseurl}{endpoint_address}' +def _add_path_evaluator_headers(headers): + """Add browser-equivalent metadata required by current website pathEvaluator requests.""" + headers.update({ + 'Origin': BASE_URL, + 'Referer': f'{BASE_URL}/browse', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'X-Netflix.browserName': 'Firefox', + 'X-Netflix.clientType': 'akira', + 'x-netflix.request.attempt': '1', + 'x-netflix.request.client.context': 'www.netflix.com' + }) + _set_header_if_value( + headers, 'X-Netflix.browserVersion', + G.LOCAL_DB.get_value('browser_info_version', '', table=TABLE_SESSION)) + _set_header_if_value( + headers, 'X-Netflix.osName', + G.LOCAL_DB.get_value('browser_info_os_name', '', table=TABLE_SESSION)) + _set_header_if_value( + headers, 'X-Netflix.osVersion', + G.LOCAL_DB.get_value('browser_info_os_version', '', table=TABLE_SESSION)) + _set_header_if_value( + headers, 'X-Netflix.uiVersion', + G.LOCAL_DB.get_value('ui_version', '', table=TABLE_SESSION)) + _set_header_if_value( + headers, 'x-netflix.request.id', + G.LOCAL_DB.get_value('request_id', '', table=TABLE_SESSION)) + website_esn = get_website_esn() + if website_esn: + headers['X-Netflix.esn'] = website_esn + headers['X-Netflix.esnPrefix'] = website_esn.rsplit('-', 1)[0] if '-' in website_esn else website_esn + + +def _graphql_headers(referer=None): + """Add browser-equivalent metadata required by current website GraphQL requests.""" + headers = { + 'Accept': '*/*', + 'Content-Type': 'application/json', + 'Origin': BASE_URL, + 'Referer': referer or f'{BASE_URL}/browse', + 'x-netflix.nq.stack': 'prod', + 'x-netflix.request.client.user.guid': G.LOCAL_DB.get_active_profile_guid() + } + _set_header_if_value( + headers, 'X-Netflix.browserVersion', + G.LOCAL_DB.get_value('browser_info_version', '', table=TABLE_SESSION)) + _set_header_if_value( + headers, 'X-Netflix.osName', + G.LOCAL_DB.get_value('browser_info_os_name', '', table=TABLE_SESSION)) + _set_header_if_value( + headers, 'X-Netflix.osVersion', + G.LOCAL_DB.get_value('browser_info_os_version', '', table=TABLE_SESSION)) + _set_header_if_value( + headers, 'X-Netflix.uiVersion', + G.LOCAL_DB.get_value('ui_version', '', table=TABLE_SESSION)) + return headers + + +def _set_header_if_value(headers, name, value): + if value: + headers[name] = value + + def _raise_api_error(decoded_response): if decoded_response.get('status', 'success') == 'error': raise APIError(decoded_response.get('message')) diff --git a/resources/lib/services/playback/action_controller.py b/resources/lib/services/playback/action_controller.py index 82083308e..1315ae5a9 100644 --- a/resources/lib/services/playback/action_controller.py +++ b/resources/lib/services/playback/action_controller.py @@ -52,6 +52,7 @@ def __init__(self, nfsession: 'NFSessionOperations', msl_handler: 'MSLHandler', self._last_player_state = {} self._is_pause_called = False self._is_av_started = False + self._is_playback_started_notified = False self._av_change_last_ts = None self._is_delayed_seek = False self._is_ads_plan = G.LOCAL_DB.get_value('is_ads_plan', None, table=TABLE_SESSION) @@ -70,6 +71,7 @@ def initialize_playback(self, **kwargs): def _initialize_am(self): self._last_player_state = {} self._is_pause_called = False + self._is_playback_started_notified = False self._av_change_last_ts = None self._is_delayed_seek = False if not self._init_data: @@ -163,6 +165,7 @@ def onNotification(self, sender, method, data): # pylint: disable=unused-argume LOG.error(traceback.format_exc()) self.is_tracking_enabled = False self._is_av_started = False + self._is_playback_started_notified = False if self._playback_tick and self._playback_tick.is_alive(): self._playback_tick.stop_join() self._playback_tick = None @@ -176,6 +179,8 @@ def on_playback_tick(self): player_state = self._get_player_state() if not player_state: return + if not self._is_playback_started_notified: + self._notify_playback_started(player_state) # If we are waiting for OnAVChange events, dont send call_on_tick otherwise will mix old/new player_state info if not self._av_change_last_ts: self._notify_all(ActionManager.call_on_tick, player_state) @@ -195,10 +200,18 @@ def _on_avchange_delayed(self, player_state): def _on_playback_started(self): player_id = _get_player_id() - self._notify_all(ActionManager.call_on_playback_started, self._get_player_state(player_id)) + self.active_player_id = player_id + player_state = self._get_player_state(player_id) + if not player_state: + LOG.warn('ActionController: delayed playback-start manager notifications until player state is available') + return + self._notify_playback_started(player_state) + + def _notify_playback_started(self, player_state): + self._notify_all(ActionManager.call_on_playback_started, player_state) + self._is_playback_started_notified = True if LOG.is_enabled and G.ADDON.getSettingBool('show_codec_info'): common.json_rpc('Input.ExecuteAction', {'action': 'codecinfo'}) - self.active_player_id = player_id def _on_playback_seek(self, time_override): if self.active_player_id is not None: @@ -228,11 +241,15 @@ def _on_playback_stopped(self): self.active_player_id = None # Immediately send the request to release the license common.run_threaded(True, self.msl_handler.release_license) - self._notify_all(ActionManager.call_on_playback_stopped, - self._last_player_state) + if self._is_playback_started_notified and self._last_player_state: + self._notify_all(ActionManager.call_on_playback_stopped, + self._last_player_state) + else: + LOG.warn('ActionController: skipped playback-stop manager notifications due to missing player state') self.action_managers = None self.init_count -= 1 self._is_av_started = False + self._is_playback_started_notified = False def _notify_all(self, notification, data=None): LOG.debug('Notifying all action managers of {} (data={})', notification.__name__, data) @@ -280,10 +297,12 @@ def _get_player_state(self, player_id=None, time_override=None): except IOError as exc: LOG.warn('_get_player_state: {}', exc) return {} - if not player_state['currentaudiostream'] and player_state['audiostreams']: + if not player_state.get('currentaudiostream') and player_state.get('audiostreams'): return {} # if audio stream has not been loaded yet, there is empty currentaudiostream - if not player_state['currentsubtitle'] and player_state['subtitles']: + if not player_state.get('currentsubtitle') and player_state.get('subtitles'): return {} # if subtitle stream has not been loaded yet, there is empty currentsubtitle + if not player_state.get('videostreams') or not player_state.get('time') or 'percentage' not in player_state: + return {} # if video stream details have not been loaded yet try: player_state['playerid'] = self.active_player_id if player_id is None else player_id # convert time dict to elapsed seconds diff --git a/resources/lib/services/playback/am_playback.py b/resources/lib/services/playback/am_playback.py index 0f61afc6b..44bafe2dc 100644 --- a/resources/lib/services/playback/am_playback.py +++ b/resources/lib/services/playback/am_playback.py @@ -13,6 +13,7 @@ import xbmcvfs import resources.lib.common as common +from resources.lib.common.cache_utils import CACHE_BOOKMARKS, CACHE_INFOLABELS from resources.lib.globals import G from resources.lib.utils.logging import LOG from .action_manager import ActionManager @@ -92,12 +93,16 @@ def on_playback_stopped(self, player_state): # falls in the part that kodi recognizes as unwatched (playcountminimumpercent 90% + no-mans land 2%) # https://kodi.wiki/view/HOW-TO:Modify_automatic_watch_and_resume_points#Settings_explained # In these cases we try change/fix manually the watched status of the video by using netflix offset data - if int(player_state['percentage']) > 92: - return - if not self.watched_threshold or not player_state['current_pts'] > self.watched_threshold: + watched_by_percent = int(player_state['percentage']) > 92 + watched_by_threshold = self.watched_threshold and player_state['current_pts'] > self.watched_threshold + if not watched_by_percent and not watched_by_threshold: return if G.ADDON.getSettingBool('sync_watched_status') and not self.is_played_from_strm: - # This have not to be applied with our custom watched status of Netflix sync, within the addon + profile_guid = G.LOCAL_DB.get_active_profile_guid() + G.SHARED_DB.set_watched_status(profile_guid, self.videoid.value, True) + G.CACHE.delete(CACHE_INFOLABELS, f'{self.videoid.value}_{G.LOCAL_DB.get_profile_config("language", "")}') + G.CACHE.delete(CACHE_BOOKMARKS, self.videoid.value) + LOG.info('Has been fixed the local watched status of the video: {}', self.videoid) return if self.is_played_from_strm: # The current video played is a STRM, then generate the path of a STRM file diff --git a/resources/lib/services/playback/am_video_events.py b/resources/lib/services/playback/am_video_events.py index a8823d181..5361cb812 100644 --- a/resources/lib/services/playback/am_video_events.py +++ b/resources/lib/services/playback/am_video_events.py @@ -8,10 +8,12 @@ See LICENSES/MIT.md for more information. """ from typing import TYPE_CHECKING +import time from resources.lib import common from resources.lib.common.cache_utils import CACHE_BOOKMARKS, CACHE_COMMON, CACHE_MANIFESTS from resources.lib.common.exceptions import InvalidVideoListTypeError +import requests.exceptions as req_exceptions from resources.lib.globals import G from resources.lib.services.nfsession.msl.msl_utils import EVENT_ENGAGE, EVENT_START, EVENT_STOP, EVENT_KEEP_ALIVE from resources.lib.utils.api_paths import build_paths, EVENT_PATHS @@ -54,7 +56,13 @@ def initialize(self, data): return if (not data['is_played_from_strm'] or (data['is_played_from_strm'] and G.ADDON.getSettingBool('sync_watched_status_library'))): - self.event_data = self._get_event_data(self.videoid) + try: + self.event_data = self._get_event_data(self.videoid) + except req_exceptions.HTTPError as exc: + if getattr(exc.response, 'status_code', None) != 404: + raise + LOG.warn('AMVideoEvents: falling back to metadata endpoint because video event metadata path returned 404') + self.event_data = self._get_event_data_metadata(self.videoid) self.event_data['videoid'] = self.videoid self.event_data['is_played_by_library'] = data['is_played_from_strm'] else: @@ -67,7 +75,7 @@ def on_playback_started(self, player_state): try: videoid_exists, list_id = self.directory_builder.get_continuewatching_videoid_exists( str(self.videoid_parent.value)) - if not videoid_exists: + if not videoid_exists and list_id: # Delete the cache of continueWatching list G.CACHE.delete(CACHE_COMMON, list_id, including_suffixes=True) # When the continueWatching context is invalidated from a refreshListByContext call @@ -202,6 +210,33 @@ def _get_event_data(self, videoid): event_data['track_id'] = videoid_data['trackIds']['value']['trackId_jaw'] return event_data + def _get_event_data_metadata(self, videoid): + parent_id = videoid.tvshowid if videoid.mediatype == common.VideoId.EPISODE else videoid.value + metadata_data = self.nfsession.get_safe( + endpoint='metadata', + params={'movieid': parent_id, '_': int(time.time() * 1000)}) + item = metadata_data.get('video', {}) + if videoid.mediatype == common.VideoId.EPISODE: + for season in item.get('seasons', []): + for episode in season.get('episodes', []): + if str(episode.get('id')) == videoid.value: + item = episode + break + else: + continue + break + bookmark = item.get('bookmark') or {} + bookmark_position = bookmark.get('offset') or item.get('bookmarkPosition') or 0 + return { + 'resume_position': bookmark_position if bookmark_position and bookmark_position > -1 else None, + 'runtime': item.get('runtime') or 0, + 'request_id': item.get('requestId') or '', + 'watched': bool(bookmark.get('watchedDate') or item.get('watched')), + 'is_in_mylist': False, + 'track_id': (item.get('trackIds') or {}).get('trackId_jawEpisode') or + (item.get('trackIds') or {}).get('trackId_jaw') or 0 + } + def _get_video_raw_data(self, videoids): """Retrieve raw data for specified video id's""" video_ids = [int(videoid.value) for videoid in videoids] diff --git a/resources/lib/utils/api_paths.py b/resources/lib/utils/api_paths.py index f3e641f4e..bb0f78f6e 100644 --- a/resources/lib/utils/api_paths.py +++ b/resources/lib/utils/api_paths.py @@ -35,7 +35,7 @@ """Predefined lambda expressions that return the number of video results within a path response dict""" -ART_PARTIAL_PATHS = [ # art moved to graphql endpoint +ART_PARTIAL_PATHS = [ ['boxarts', [ART_SIZE_SD, ART_SIZE_FHD, ART_SIZE_POSTER], 'jpg', 'value'], ['interestingMoment', [ART_SIZE_SD, ART_SIZE_FHD], 'jpg', 'value'], ['artWorkByType', 'LOGO_BRANDED_HORIZONTAL', '_550x124', 'png', 'value'], # 11/05/2020 same img of bb2OGLogo @@ -48,16 +48,16 @@ VIDEO_LIST_PARTIAL_PATHS = [ - [['summary', 'title', 'synopsis', 'queue', 'inRemindMeList', - 'episodeCount', 'maturity', 'runtime', 'seasonCount', 'availability', 'trackIds', + [['requestId', 'summary', 'title', 'synopsis', 'regularSynopsis', 'evidence', 'queue', 'inRemindMeList', + 'episodeCount', 'info', 'maturity', 'runtime', 'seasonCount', 'availability', 'trackIds', 'releaseYear', 'userRating', 'numSeasonsLabel', 'bookmarkPosition', 'creditsOffset', - 'delivery', 'availability', 'itemSummary']] - #,[['genres', 'tags', 'creators', 'directors', 'cast'], - # {'from': 0, 'to': 10}, ['id', 'name']] -]# + ART_PARTIAL_PATHS + 'dpSupplementalMessage', 'watched', 'delivery', 'sequiturEvidence', 'promoVideo', 'availability', 'itemSummary']], + [['genres', 'tags', 'creators', 'directors', 'cast'], + {'from': 0, 'to': 10}, ['id', 'name']] +] + ART_PARTIAL_PATHS VIDEO_LIST_BASIC_PARTIAL_PATHS = [ - [['queue', 'summary']] + [['title', 'queue', 'watched', 'summary', 'type', 'id']] ] GENRE_PARTIAL_PATHS = [ @@ -70,20 +70,20 @@ SEASONS_PARTIAL_PATHS = [ ['seasonList', RANGE_PLACEHOLDER, 'summary'], ['title'] -]# + ART_PARTIAL_PATHS +] + ART_PARTIAL_PATHS EPISODES_PARTIAL_PATHS = [ - [['summary', 'synopsis', 'title', 'runtime', 'releaseYear', 'queue', - 'maturity', 'userRating', 'bookmarkPosition', 'creditsOffset', - 'delivery', 'trackIds', 'availability']], - [['genres', 'creators', 'directors', 'cast'], + [['requestId', 'summary', 'synopsis', 'regularSynopsis', 'title', 'runtime', 'releaseYear', 'queue', + 'info', 'maturity', 'userRating', 'bookmarkPosition', 'creditsOffset', + 'watched', 'delivery', 'trackIds', 'availability']], + [['genres', 'tags', 'creators', 'directors', 'cast'], {'from': 0, 'to': 10}, ['id', 'name']] -]# + ART_PARTIAL_PATHS +] + ART_PARTIAL_PATHS TRAILER_PARTIAL_PATHS = [ - [['availability', 'summary', 'synopsis', 'title', 'trackIds', 'delivery', 'runtime', + [['availability', 'summary', 'synopsis', 'regularSynopsis', 'title', 'trackIds', 'delivery', 'runtime', 'bookmarkPosition', 'creditsOffset']] -]# + ART_PARTIAL_PATHS +] + ART_PARTIAL_PATHS EVENT_PATHS = [ [['requestId', 'title', 'runtime', 'queue', 'bookmarkPosition', 'watched', 'trackIds']] @@ -112,7 +112,9 @@ ('Duration', ['runtime', 'value']), # 'trailer' add the trailer button support to 'Information' window of ListItem, can be used from custom Kodi skins # to reproduce a background promo video when a ListItem is selected + ('Trailer', ['trailerUrl', 'value']), ('Trailer', ['promoVideo', 'value', 'id']), + ('Trailer', ['promoVideo', 'value', 'videoId']), # ListItem.DateAdded: Removed for now, the actual use of this property for tvshow ListItem type is not clear, # the documentation says "date of adding in the library", but kodi developers say that # is used as the latest update date @@ -124,8 +126,7 @@ 'Season': lambda s_value: _convert_season(s_value), 'Rating': lambda r: r / 10, 'PlayCount': lambda w: int(w), - 'Trailer': lambda video_id: common.build_url(pathitems=[common.VideoId.SUPPLEMENTAL, str(video_id)], - mode=G.MODE_PLAY), + 'Trailer': lambda video_id: _convert_trailer(video_id), 'DateAdded': lambda ats: common.strf_timestamp(int(ats / 1000), '%Y-%m-%d %H:%M:%S') } @@ -144,6 +145,13 @@ def _convert_season(value): return int(''.join([n for n in value if n.isdigit()] or '0')) +def _convert_trailer(value): + if isinstance(value, str) and (value.startswith('http') or value.startswith('plugin://')): + return value + return common.build_url(pathitems=[common.VideoId.SUPPLEMENTAL, str(value)], + mode=G.MODE_PLAY) + + def build_paths(base_path, partial_paths): """Build a list of full paths by concatenating each partial path with the base path""" paths = [base_path + partial_path for partial_path in partial_paths] diff --git a/resources/lib/utils/api_requests.py b/resources/lib/utils/api_requests.py index c6cd1ce5a..faec1dece 100644 --- a/resources/lib/utils/api_requests.py +++ b/resources/lib/utils/api_requests.py @@ -18,6 +18,22 @@ from ..database.db_utils import TABLE_SESSION +MY_LIST_GRAPHQL_MUTATIONS = { + 'add': { + 'operation_name': 'AddToPlaylist', + 'operation_id': '8d985f07-2117-4add-980c-b83895549d1c', + 'response_key': 'addEntityToPlaylist', + 'expected_state': True + }, + 'remove': { + 'operation_name': 'RemoveFromPlaylist', + 'operation_id': '11403da1-d6fd-4cd7-9ad7-892554b70047', + 'response_key': 'removeEntityFromPlaylist', + 'expected_state': False + } +} + + def logout(): """Logout of the current account""" common.make_call('logout') @@ -36,19 +52,11 @@ def login(ask_credentials=True): is_login_with_credentials = ui.show_yesno_dialog('Login', common.get_local_string(30340), yeslabel=common.get_local_string(30341), nolabel=common.get_local_string(30342)) - # if is_login_with_credentials: - # credentials = {'credentials': ui.ask_credentials()} - if is_login_with_credentials: - # The login page is changed now part of HTML seem protected by reCaptcha - # in the HTML page the reactContext data is added after the reCaptcha checks so at the moment - # it is not accessible by requesting the login page through python script, - # this prevents us to get the authURL code needed to perform the login request - ui.show_ok_dialog('Login', - 'Due to new website protections at moment the login with credentials is not available.') - is_login_with_credentials = False + if is_login_with_credentials: + credentials = {'credentials': ui.ask_credentials()} if is_login_with_credentials: - if common.make_call('login', credentials): + if common.make_call('login', credentials, timeout=common.IPC_TIMEOUT_SECS_LOGIN): is_success = True else: data = common.run_nf_authentication_key() @@ -152,20 +160,26 @@ def update_remindme(operation, videoid, trackid): @measure_exec_time_decorator() def update_my_list(videoid, operation, params): """Call API to add / remove videos to my list""" - if params['trackid'] == 'None': - raise ErrorMsg('Unable update my list, trackid not found.') + mutation = MY_LIST_GRAPHQL_MUTATIONS.get(operation) + if not mutation: + raise APIError(f'Unsupported my list operation: {operation}') + LOG.debug('My List: {} {}', operation, videoid) + variables = {'entityId': str(videoid.value)} + trackid = params.get('trackid') + if trackid and trackid != 'None': + variables['trackId'] = str(trackid) + response = common.make_call( - 'post_safe', - {'endpoint': 'playlistop', - 'data': { - 'lolomoId': 'unknown', - 'operation': operation, - 'videoId': int(videoid.value), - 'trackId': int(params['trackid']), - 'skipRootInvalidation': True, - }}) - if response.get('status') != 'success': + 'post_graphql', + {'operation_name': mutation['operation_name'], + 'operation_id': mutation['operation_id'], + 'variables': variables, + 'referer': f'https://www.netflix.com/title/{videoid.value}'}) + entity = (response.get('data', {}) + .get(mutation['response_key'], {}) + .get('entity', {})) + if entity.get('isInPlaylist') != mutation['expected_state']: LOG.debug('update_my_list response: {}', response) raise APIError('Unable update my list, an error occurred in the request.') _update_mylist_cache(videoid, operation, params) @@ -192,12 +206,11 @@ def _update_mylist_cache(videoid, operation, params): except CacheMiss: pass else: - common.make_call('add_videoids_to_video_list_cache', {'cache_bucket': cache_utils.CACHE_MYLIST, - 'cache_identifier': mylist_identifier, - 'video_ids': [videoid.value]}) + G.CACHE.delete(cache_utils.CACHE_MYLIST, mylist_identifier) try: my_list_videoids = G.CACHE.get(cache_utils.CACHE_MYLIST, 'my_list_items') - my_list_videoids.append(videoid) + if videoid not in my_list_videoids: + my_list_videoids.append(videoid) G.CACHE.add(cache_utils.CACHE_MYLIST, 'my_list_items', my_list_videoids) except CacheMiss: pass diff --git a/resources/lib/utils/data_types.py b/resources/lib/utils/data_types.py index 06d309152..8f3b083b1 100644 --- a/resources/lib/utils/data_types.py +++ b/resources/lib/utils/data_types.py @@ -335,10 +335,13 @@ def _get_titles(videos): def _filterout_loco_contexts(root_id, data, contexts): """Deletes from the data all records related to the specified contexts""" - if not data: - return - lists = data['lists'] - for key in list(lists.keys()): - context = lists[key].get('componentSummary', {}).get('value', {}).get('context') - if context in contexts: - del lists[key] + total_items = data['locos'][root_id]['componentSummary'].get('value', {}).get('length', 0) + for index in range(total_items - 1, -1, -1): + row = data['locos'][root_id].get(str(index)) + if not row: + continue + list_id = row['value'][1] + if not data['lists'][list_id]['componentSummary'].get('value', {}).get('context') in contexts: + continue + del data['lists'][list_id] + del data['locos'][root_id][str(index)] diff --git a/resources/lib/utils/website.py b/resources/lib/utils/website.py index f1471bd5a..ba2265d84 100644 --- a/resources/lib/utils/website.py +++ b/resources/lib/utils/website.py @@ -278,6 +278,72 @@ def validate_login(react_context): raise WebsiteParsingError(error_msg) from exc +def decode_javascript_string(value): + """Decode a JavaScript string literal without corrupting decoded Unicode.""" + decoded = [] + index = 0 + value_length = len(value) + simple_escapes = { + "'": "'", '"': '"', '\\': '\\', '/': '/', + 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', 'v': '\v', '0': '\0' + } + while index < value_length: + character = value[index] + if character != '\\': + decoded.append(character) + index += 1 + continue + index += 1 + if index >= value_length: + decoded.append('\\') + break + escape = value[index] + if escape == 'u' and index + 4 < value_length: + hex_value = value[index + 1:index + 5] + try: + codepoint = int(hex_value, 16) + except ValueError: + pass + else: + index += 5 + if (0xD800 <= codepoint <= 0xDBFF and + index + 5 < value_length and value[index:index + 2] == '\\u'): + low_hex_value = value[index + 2:index + 6] + try: + low_codepoint = int(low_hex_value, 16) + except ValueError: + pass + else: + if 0xDC00 <= low_codepoint <= 0xDFFF: + codepoint = 0x10000 + ((codepoint - 0xD800) << 10) + (low_codepoint - 0xDC00) + index += 6 + decoded.append(chr(codepoint)) + continue + elif escape == 'x' and index + 2 < value_length: + hex_value = value[index + 1:index + 3] + try: + decoded.append(chr(int(hex_value, 16))) + index += 3 + continue + except ValueError: + pass + elif escape in simple_escapes: + decoded.append(simple_escapes[escape]) + index += 1 + continue + elif escape == '\n': + index += 1 + continue + elif escape == '\r': + index += 1 + if index < value_length and value[index] == '\n': + index += 1 + continue + decoded.append(escape) + index += 1 + return ''.join(decoded) + + @measure_exec_time_decorator(is_immediate=True) def extract_json(content, name): """Extract json from netflix content page""" @@ -292,7 +358,7 @@ def extract_json(content, name): json_str_replace = json_str_replace.replace(r'\n', r'\\n') # Escape line feed json_str_replace = json_str_replace.replace(r'\t', r'\\t') # Escape tab json_str_replace = json_str_replace.replace(r'\p', r'/p') # Unicode property not supported, we change slash to avoid unescape it - json_str_replace = json_str_replace.encode().decode('unicode_escape') # Decode the string as unicode + json_str_replace = decode_javascript_string(json_str_replace) json_str_replace = sub(r'\\(?!["])', r'\\\\', json_str_replace) # Escape backslash (only when is not followed by double quotation marks \") return json.loads(json_str_replace) except Exception as exc: # pylint: disable=broad-except From e888205306e9c3beef4652ca4fb4d2c5057de532 Mon Sep 17 00:00:00 2001 From: Breezyslasher <167659924+Breezyslasher@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:44:44 -0400 Subject: [PATCH 4/4] Add files via upload --- resources/lib/database/db_shared.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/resources/lib/database/db_shared.py b/resources/lib/database/db_shared.py index da4ea1123..c37cf7d46 100644 --- a/resources/lib/database/db_shared.py +++ b/resources/lib/database/db_shared.py @@ -210,7 +210,8 @@ def movie_id_exists(self, movieid): """Return True if a movie id exists""" query = 'SELECT EXISTS(SELECT 1 FROM video_lib_movies WHERE MovieID = ?)' cur = self._execute_query(query, (movieid,)) - return bool(cur.fetchone()[0]) + result = cur.fetchone() + return bool(result and result[0]) @db_base_mysql.handle_connection @db_base_sqlite.handle_connection @@ -218,7 +219,8 @@ def tvshow_id_exists(self, tvshowid): """Return True if a tvshow id exists""" query = 'SELECT EXISTS(SELECT 1 FROM video_lib_tvshows WHERE TvShowID = ?)' cur = self._execute_query(query, (tvshowid,)) - return bool(cur.fetchone()[0]) + result = cur.fetchone() + return bool(result and result[0]) @db_base_mysql.handle_connection @db_base_sqlite.handle_connection @@ -231,7 +233,8 @@ def season_id_exists(self, tvshowid, seasonid): 'ON video_lib_seasons.TvShowID = video_lib_tvshows.TvShowID ' 'WHERE video_lib_tvshows.TvShowID = ? AND video_lib_seasons.SeasonID = ?)') cur = self._execute_query(query, (tvshowid, seasonid)) - return bool(cur.fetchone()[0]) + result = cur.fetchone() + return bool(result and result[0]) @db_base_mysql.handle_connection @db_base_sqlite.handle_connection @@ -248,7 +251,8 @@ def episode_id_exists(self, tvshowid, seasonid, episodeid): 'video_lib_seasons.SeasonID = ? AND ' 'video_lib_episodes.EpisodeID = ?)') cur = self._execute_query(query, (tvshowid, seasonid, episodeid)) - return bool(cur.fetchone()[0]) + result = cur.fetchone() + return bool(result and result[0]) @db_base_mysql.handle_connection @db_base_sqlite.handle_connection