diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index 1cb4cf9a..31ac2741 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -61,6 +61,7 @@ The list views return a paginate object with the following keys: - prev_pages: *array of ints* of previous pages, if available - current_page: *int* current page from query - max_hits: *bool* if max of 10k results is reached +- params: *str* additional url encoded query parameters - last_page: *int* of last page link - next_pages: *array of ints* of next pages - total_hits: *int* total results @@ -169,6 +170,7 @@ GET /api/download/ Parameter: - filter: pending, ignore +- channel: channel-id ### Add list of videos to download queue POST /api/download/ diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 94e6af6d..70d2485e 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -53,9 +53,7 @@ class ApiBaseView(APIView): def initiate_pagination(self, request): """set initial pagination values""" - user_id = request.user.id - page_get = int(request.GET.get("page", 0)) - self.pagination_handler = Pagination(page_get, user_id) + self.pagination_handler = Pagination(request) self.data.update( { "size": self.pagination_handler.pagination["page_size"], @@ -368,13 +366,23 @@ class DownloadApiListView(ApiBaseView): """get request""" query_filter = request.GET.get("filter", False) self.data.update({"sort": [{"timestamp": {"order": "asc"}}]}) + + must_list = [] if query_filter: if query_filter not in self.valid_filter: message = f"invalid url query filder: {query_filter}" print(message) return Response({"message": message}, status=400) - self.data["query"] = {"term": {"status": {"value": query_filter}}} + must_list.append({"term": {"status": {"value": query_filter}}}) + + filter_channel = request.GET.get("channel", False) + if filter_channel: + must_list.append( + {"term": {"channel_id": {"value": filter_channel}}} + ) + + self.data["query"] = {"bool": {"must": must_list}} self.get_document_list(request) return Response(self.response) diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index f1f4ac2f..13bd49be 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -246,7 +246,7 @@ class PendingList(PendingIndex): print(f"{youtube_id}: skipping premium video, id not matching") return False # stop if video is streaming live now - if vid["is_live"]: + if vid["live_status"] in ["is_upcoming", "is_live"]: return False return self._parse_youtube_details(vid) diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index 7f8a1676..95ff358a 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -63,6 +63,7 @@ class ChannelSubscription: for idx, channel in enumerate(all_channels): channel_id = channel["channel_id"] + print(f"{channel_id}: find missing videos.") last_videos = self.get_last_youtube_videos(channel_id) if last_videos: diff --git a/tubearchivist/home/src/index/generic.py b/tubearchivist/home/src/index/generic.py index 15e12648..099c5d45 100644 --- a/tubearchivist/home/src/index/generic.py +++ b/tubearchivist/home/src/index/generic.py @@ -73,16 +73,25 @@ class Pagination: figure out the pagination based on page size and total_hits """ - def __init__(self, page_get, user_id, search_get=False): - self.user_id = user_id + def __init__(self, request): + self.request = request + self.page_get = False + self.params = False + self.get_params() self.page_size = self.get_page_size() - self.page_get = page_get - self.search_get = search_get self.pagination = self.first_guess() + def get_params(self): + """process url query parameters""" + query_dict = self.request.GET.copy() + self.page_get = int(query_dict.get("page", 0)) + + _ = query_dict.pop("page", False) + self.params = query_dict.urlencode() + def get_page_size(self): """get default or user modified page_size""" - key = f"{self.user_id}:page_size" + key = f"{self.request.user.id}:page_size" page_size = RedisArchivist().get_message(key)["status"] if not page_size: config = AppConfig().config @@ -108,9 +117,9 @@ class Pagination: "prev_pages": prev_pages, "current_page": page_get, "max_hits": False, + "params": self.params, } - if self.search_get: - pagination.update({"search_get": self.search_get}) + return pagination def validate(self, total_hits): diff --git a/tubearchivist/home/templates/home/base.html b/tubearchivist/home/templates/home/base.html index 83b41f93..5e539956 100644 --- a/tubearchivist/home/templates/home/base.html +++ b/tubearchivist/home/templates/home/base.html @@ -79,16 +79,16 @@
diff --git a/tubearchivist/home/templates/home/channel_id_about.html b/tubearchivist/home/templates/home/channel_id_about.html index d3bc9b69..5c6ac892 100644 --- a/tubearchivist/home/templates/home/channel_id_about.html +++ b/tubearchivist/home/templates/home/channel_id_about.html @@ -10,6 +10,9 @@

Videos

Playlists

About

 + {% if has_pending %} +

Downloads

 + {% endif %}
diff --git a/tubearchivist/home/templates/home/channel_id_playlist.html b/tubearchivist/home/templates/home/channel_id_playlist.html index b7f5891c..9fd51b58 100644 --- a/tubearchivist/home/templates/home/channel_id_playlist.html +++ b/tubearchivist/home/templates/home/channel_id_playlist.html @@ -10,6 +10,9 @@

Videos

Playlists

About

 + {% if has_pending %} +

Downloads

 + {% endif %}
diff --git a/tubearchivist/home/templates/home/downloads.html b/tubearchivist/home/templates/home/downloads.html index b3a3a706..85d61e7c 100644 --- a/tubearchivist/home/templates/home/downloads.html +++ b/tubearchivist/home/templates/home/downloads.html @@ -3,7 +3,7 @@ {% block content %}
-

Downloads

+

Downloads {% if channel_filter_id %} for {{ channel_filter_name }}{% endif %}

@@ -55,7 +55,7 @@ list view
-

Total videos: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %}

+

Total videos: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %} {% if channel_filter_id %} - from channel {{ channel_filter_name }}{% endif %}

diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 55b6918c..9d44399d 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -31,7 +31,7 @@ from home.src.frontend.forms import ( UserSettingsForm, ) from home.src.frontend.searching import SearchHandler -from home.src.index.channel import channel_overwrites +from home.src.index.channel import YoutubeChannel, channel_overwrites from home.src.index.generic import Pagination from home.src.index.playlist import YoutubePlaylist from home.src.ta.config import AppConfig, ScheduleBuilder @@ -237,14 +237,10 @@ class ArchivistResultsView(ArchivistViewConfig): def initiate_vars(self, request): """search in es for vidoe hits""" - page_get = int(request.GET.get("page", 0)) self.user_id = request.user.id self.config_builder(self.user_id) self.search_get = request.GET.get("search", False) - search_encoded = self._url_encode(self.search_get) - self.pagination_handler = Pagination( - page_get=page_get, user_id=self.user_id, search_get=search_encoded - ) + self.pagination_handler = Pagination(request) self.sort_by = self._sort_by_overwrite() self._initial_data() @@ -362,7 +358,7 @@ class DownloadView(ArchivistResultsView): def get(self, request): """handle get request""" self.initiate_vars(request) - self._update_view_data() + self._update_view_data(request) self.find_results() self.context.update( { @@ -372,15 +368,33 @@ class DownloadView(ArchivistResultsView): ) return render(request, "home/downloads.html", self.context) - def _update_view_data(self): + def _update_view_data(self, request): """update downloads view specific data dict""" if self.context["show_ignored_only"]: filter_view = "ignore" else: filter_view = "pending" + + must_list = [{"term": {"status": {"value": filter_view}}}] + + channel_filter = request.GET.get("channel", False) + if channel_filter: + must_list.append( + {"term": {"channel_id": {"value": channel_filter}}} + ) + + channel = YoutubeChannel(channel_filter) + channel.get_from_es() + self.context.update( + { + "channel_filter_id": channel_filter, + "channel_filter_name": channel.json_data["channel_name"], + } + ) + self.data.update( { - "query": {"term": {"status": {"value": filter_view}}}, + "query": {"bool": {"must": must_list}}, "sort": [{"timestamp": {"order": "asc"}}], } ) @@ -414,7 +428,37 @@ class DownloadView(ArchivistResultsView): return redirect("downloads", permanent=True) -class ChannelIdView(ArchivistResultsView): +class ChannelIdBaseView(ArchivistResultsView): + """base class for all channel-id views""" + + def get_channel_meta(self, channel_id): + """get metadata for channel""" + path = f"ta_channel/_doc/{channel_id}" + response, _ = ElasticWrap(path).get() + channel_info = SearchProcess(response).process() + + return channel_info + + def channel_has_pending(self, channel_id): + """check if channel has pending videos in queue""" + path = "ta_download/_search" + data = { + "size": 1, + "query": { + "bool": { + "must": [ + {"term": {"status": {"value": "pending"}}}, + {"term": {"channel_id": {"value": channel_id}}}, + ] + } + }, + } + response, _ = ElasticWrap(path).get(data=data) + + self.context.update({"has_pending": bool(response["hits"]["hits"])}) + + +class ChannelIdView(ChannelIdBaseView): """resolves to /channel// display single channel page from channel_id """ @@ -428,6 +472,7 @@ class ChannelIdView(ArchivistResultsView): self._update_view_data(channel_id) self.find_results() self.match_progress() + self.channel_has_pending(channel_id) if self.context["results"]: channel_info = self.context["results"][0]["source"]["channel"] @@ -478,7 +523,7 @@ class ChannelIdView(ArchivistResultsView): return redirect("channel_id", channel_id, permanent=True) -class ChannelIdAboutView(ArchivistResultsView): +class ChannelIdAboutView(ChannelIdBaseView): """resolves to /channel//about/ show metadata, handle per channel conf """ @@ -488,6 +533,7 @@ class ChannelIdAboutView(ArchivistResultsView): def get(self, request, channel_id): """handle get request""" self.initiate_vars(request) + self.channel_has_pending(channel_id) path = f"ta_channel/_doc/{channel_id}" response, _ = ElasticWrap(path).get() @@ -521,7 +567,7 @@ class ChannelIdAboutView(ArchivistResultsView): return redirect("channel_id_about", channel_id, permanent=True) -class ChannelIdPlaylistView(ArchivistResultsView): +class ChannelIdPlaylistView(ChannelIdBaseView): """resolves to /channel//playlist/ show all playlists of channel """ @@ -534,8 +580,9 @@ class ChannelIdPlaylistView(ArchivistResultsView): self.initiate_vars(request) self._update_view_data(channel_id) self.find_results() + self.channel_has_pending(channel_id) - channel_info = self._get_channel_meta(channel_id) + channel_info = self.get_channel_meta(channel_id) channel_name = channel_info["channel_name"] self.context.update( { @@ -556,14 +603,6 @@ class ChannelIdPlaylistView(ArchivistResultsView): self.data["query"] = {"bool": {"must": must_list}} - def _get_channel_meta(self, channel_id): - """get metadata for channel""" - path = f"ta_channel/_doc/{channel_id}" - response, _ = ElasticWrap(path).get() - channel_info = SearchProcess(response).process() - - return channel_info - class ChannelView(ArchivistResultsView): """resolves to /channel/ diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index e556893b..30a34845 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,13 +1,13 @@ beautifulsoup4==4.11.1 celery==5.2.7 -Django==4.0.6 +Django==4.1.2 django-auth-ldap==4.1.0 django-cors-headers==3.13.0 -djangorestframework==3.13.1 +djangorestframework==3.14.0 Pillow==9.2.0 redis==4.3.4 requests==2.28.1 ryd-client==0.0.6 uWSGI==2.0.20 whitenoise==6.2.0 -yt_dlp==2022.9.1 +yt_dlp==2022.10.4 diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index 178ecec6..7e375c17 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -140,7 +140,7 @@ function toggleCheckbox(checkbox) { var payload = JSON.stringify(payloadDict); sendPost(payload); setTimeout(function(){ - var currPage = window.location.pathname; + var currPage = window.location.pathname + window.location.search; window.location.replace(currPage); return false; }, 500);