From c825e67f69ce5e0f198ff8821bc0d7c5a33d881d Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 30 Sep 2022 18:02:44 +0200 Subject: [PATCH 1/8] bump django and restframework --- tubearchivist/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index e556893b..aab21706 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,9 +1,9 @@ beautifulsoup4==4.11.1 celery==5.2.7 -Django==4.0.6 +Django==4.1.1 django-auth-ldap==4.1.0 django-cors-headers==3.13.0 -djangorestframework==3.13.1 +djangorestframework==3.14.0 Pillow==9.2.0 redis==4.3.4 requests==2.28.1 From c9e936da21663a35c68d5dfca6a3f4010b1dbe3a Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 5 Oct 2022 15:46:44 +0200 Subject: [PATCH 2/8] bump libraries --- tubearchivist/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index aab21706..30a34845 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,6 +1,6 @@ beautifulsoup4==4.11.1 celery==5.2.7 -Django==4.1.1 +Django==4.1.2 django-auth-ldap==4.1.0 django-cors-headers==3.13.0 djangorestframework==3.14.0 @@ -10,4 +10,4 @@ requests==2.28.1 ryd-client==0.0.6 uWSGI==2.0.20 whitenoise==6.2.0 -yt_dlp==2022.9.1 +yt_dlp==2022.10.4 From fbb52dc93f9fbb12650654e3626d470916d06ba1 Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 5 Oct 2022 15:47:17 +0200 Subject: [PATCH 3/8] implement basic channel query string for downloads page --- tubearchivist/home/views.py | 15 ++++++++++++--- tubearchivist/static/script.js | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 55b6918c..8533585b 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -362,7 +362,7 @@ class DownloadView(ArchivistResultsView): def get(self, request): """handle get request""" self.initiate_vars(request) - self._update_view_data() + self._update_view_data(request) self.find_results() self.context.update( { @@ -372,15 +372,24 @@ class DownloadView(ArchivistResultsView): ) return render(request, "home/downloads.html", self.context) - def _update_view_data(self): + def _update_view_data(self, request): """update downloads view specific data dict""" if self.context["show_ignored_only"]: filter_view = "ignore" else: filter_view = "pending" + + must_list = [{"term": {"status": {"value": filter_view}}}] + + channel_filter = request.GET.get("channel", False) + if channel_filter: + must_list.append( + {"term": {"channel_id": {"value": channel_filter}}} + ) + self.data.update( { - "query": {"term": {"status": {"value": filter_view}}}, + "query": {"bool": {"must": must_list}}, "sort": [{"timestamp": {"order": "asc"}}], } ) diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index 178ecec6..7e375c17 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -140,7 +140,7 @@ function toggleCheckbox(checkbox) { var payload = JSON.stringify(payloadDict); sendPost(payload); setTimeout(function(){ - var currPage = window.location.pathname; + var currPage = window.location.pathname + window.location.search; window.location.replace(currPage); return false; }, 500); From a8c5773f816ea3cdf53b4f530a50650cd3abb556 Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 5 Oct 2022 16:12:58 +0200 Subject: [PATCH 4/8] fix is_live status check before adding to queue --- tubearchivist/home/src/download/queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index f1f4ac2f..13bd49be 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -246,7 +246,7 @@ class PendingList(PendingIndex): print(f"{youtube_id}: skipping premium video, id not matching") return False # stop if video is streaming live now - if vid["is_live"]: + if vid["live_status"] in ["is_upcoming", "is_live"]: return False return self._parse_youtube_details(vid) From 6eee762d3a12984f412337a2dea714fa1e2c3139 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 14 Oct 2022 11:27:27 +0700 Subject: [PATCH 5/8] add status for sub refresh --- tubearchivist/home/src/download/subscriptions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index 7f8a1676..95ff358a 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -63,6 +63,7 @@ class ChannelSubscription: for idx, channel in enumerate(all_channels): channel_id = channel["channel_id"] + print(f"{channel_id}: find missing videos.") last_videos = self.get_last_youtube_videos(channel_id) if last_videos: From 9dfd967a32441c9392a74b4018a38512933fff10 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 17 Oct 2022 13:29:21 +0700 Subject: [PATCH 6/8] implement downloads filter per channel --- .../home/templates/home/channel_id.html | 3 + .../home/templates/home/channel_id_about.html | 3 + .../templates/home/channel_id_playlist.html | 3 + .../home/templates/home/downloads.html | 4 +- tubearchivist/home/views.py | 60 +++++++++++++++---- 5 files changed, 58 insertions(+), 15 deletions(-) diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 1e5b1b42..17d26db9 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -10,6 +10,9 @@

Videos

Playlists

About

 + {% if has_pending %} +

Downloads

 + {% endif %}
diff --git a/tubearchivist/home/templates/home/channel_id_about.html b/tubearchivist/home/templates/home/channel_id_about.html index d3bc9b69..5c6ac892 100644 --- a/tubearchivist/home/templates/home/channel_id_about.html +++ b/tubearchivist/home/templates/home/channel_id_about.html @@ -10,6 +10,9 @@

Videos

Playlists

About

 + {% if has_pending %} +

Downloads

 + {% endif %}
diff --git a/tubearchivist/home/templates/home/channel_id_playlist.html b/tubearchivist/home/templates/home/channel_id_playlist.html index b7f5891c..9fd51b58 100644 --- a/tubearchivist/home/templates/home/channel_id_playlist.html +++ b/tubearchivist/home/templates/home/channel_id_playlist.html @@ -10,6 +10,9 @@

Videos

Playlists

About

 + {% if has_pending %} +

Downloads

 + {% endif %}
diff --git a/tubearchivist/home/templates/home/downloads.html b/tubearchivist/home/templates/home/downloads.html index b3a3a706..85d61e7c 100644 --- a/tubearchivist/home/templates/home/downloads.html +++ b/tubearchivist/home/templates/home/downloads.html @@ -3,7 +3,7 @@ {% block content %}
-

Downloads

+

Downloads {% if channel_filter_id %} for {{ channel_filter_name }}{% endif %}

@@ -55,7 +55,7 @@ list view
-

Total videos: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %}

+

Total videos: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %} {% if channel_filter_id %} - from channel {{ channel_filter_name }}{% endif %}

diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 8533585b..29ee565c 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -31,7 +31,7 @@ from home.src.frontend.forms import ( UserSettingsForm, ) from home.src.frontend.searching import SearchHandler -from home.src.index.channel import channel_overwrites +from home.src.index.channel import YoutubeChannel, channel_overwrites from home.src.index.generic import Pagination from home.src.index.playlist import YoutubePlaylist from home.src.ta.config import AppConfig, ScheduleBuilder @@ -387,6 +387,15 @@ class DownloadView(ArchivistResultsView): {"term": {"channel_id": {"value": channel_filter}}} ) + channel = YoutubeChannel(channel_filter) + channel.get_from_es() + self.context.update( + { + "channel_filter_id": channel_filter, + "channel_filter_name": channel.json_data["channel_name"], + } + ) + self.data.update( { "query": {"bool": {"must": must_list}}, @@ -423,7 +432,37 @@ class DownloadView(ArchivistResultsView): return redirect("downloads", permanent=True) -class ChannelIdView(ArchivistResultsView): +class ChannelIdBaseView(ArchivistResultsView): + """base class for all channel-id views""" + + def get_channel_meta(self, channel_id): + """get metadata for channel""" + path = f"ta_channel/_doc/{channel_id}" + response, _ = ElasticWrap(path).get() + channel_info = SearchProcess(response).process() + + return channel_info + + def channel_has_pending(self, channel_id): + """check if channel has pending videos in queue""" + path = "ta_download/_search" + data = { + "size": 1, + "query": { + "bool": { + "must": [ + {"term": {"status": {"value": "pending"}}}, + {"term": {"channel_id": {"value": channel_id}}}, + ] + } + }, + } + response, _ = ElasticWrap(path).get(data=data) + + self.context.update({"has_pending": bool(response["hits"]["hits"])}) + + +class ChannelIdView(ChannelIdBaseView): """resolves to /channel// display single channel page from channel_id """ @@ -437,6 +476,7 @@ class ChannelIdView(ArchivistResultsView): self._update_view_data(channel_id) self.find_results() self.match_progress() + self.channel_has_pending(channel_id) if self.context["results"]: channel_info = self.context["results"][0]["source"]["channel"] @@ -487,7 +527,7 @@ class ChannelIdView(ArchivistResultsView): return redirect("channel_id", channel_id, permanent=True) -class ChannelIdAboutView(ArchivistResultsView): +class ChannelIdAboutView(ChannelIdBaseView): """resolves to /channel//about/ show metadata, handle per channel conf """ @@ -497,6 +537,7 @@ class ChannelIdAboutView(ArchivistResultsView): def get(self, request, channel_id): """handle get request""" self.initiate_vars(request) + self.channel_has_pending(channel_id) path = f"ta_channel/_doc/{channel_id}" response, _ = ElasticWrap(path).get() @@ -530,7 +571,7 @@ class ChannelIdAboutView(ArchivistResultsView): return redirect("channel_id_about", channel_id, permanent=True) -class ChannelIdPlaylistView(ArchivistResultsView): +class ChannelIdPlaylistView(ChannelIdBaseView): """resolves to /channel//playlist/ show all playlists of channel """ @@ -543,8 +584,9 @@ class ChannelIdPlaylistView(ArchivistResultsView): self.initiate_vars(request) self._update_view_data(channel_id) self.find_results() + self.channel_has_pending(channel_id) - channel_info = self._get_channel_meta(channel_id) + channel_info = self.get_channel_meta(channel_id) channel_name = channel_info["channel_name"] self.context.update( { @@ -565,14 +607,6 @@ class ChannelIdPlaylistView(ArchivistResultsView): self.data["query"] = {"bool": {"must": must_list}} - def _get_channel_meta(self, channel_id): - """get metadata for channel""" - path = f"ta_channel/_doc/{channel_id}" - response, _ = ElasticWrap(path).get() - channel_info = SearchProcess(response).process() - - return channel_info - class ChannelView(ArchivistResultsView): """resolves to /channel/ From bd7cdb39420b3f6017056d9464d05c45adaa451e Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 17 Oct 2022 18:40:20 +0700 Subject: [PATCH 7/8] append query parameters to pagination --- tubearchivist/api/README.md | 1 + tubearchivist/api/views.py | 4 +--- tubearchivist/home/src/index/generic.py | 23 ++++++++++++++------- tubearchivist/home/templates/home/base.html | 16 +++++++------- tubearchivist/home/views.py | 6 +----- 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index 1cb4cf9a..4e75c568 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -61,6 +61,7 @@ The list views return a paginate object with the following keys: - prev_pages: *array of ints* of previous pages, if available - current_page: *int* current page from query - max_hits: *bool* if max of 10k results is reached +- params: *str* additional url encoded query parameters - last_page: *int* of last page link - next_pages: *array of ints* of next pages - total_hits: *int* total results diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 94e6af6d..fc822f5d 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -53,9 +53,7 @@ class ApiBaseView(APIView): def initiate_pagination(self, request): """set initial pagination values""" - user_id = request.user.id - page_get = int(request.GET.get("page", 0)) - self.pagination_handler = Pagination(page_get, user_id) + self.pagination_handler = Pagination(request) self.data.update( { "size": self.pagination_handler.pagination["page_size"], diff --git a/tubearchivist/home/src/index/generic.py b/tubearchivist/home/src/index/generic.py index 15e12648..099c5d45 100644 --- a/tubearchivist/home/src/index/generic.py +++ b/tubearchivist/home/src/index/generic.py @@ -73,16 +73,25 @@ class Pagination: figure out the pagination based on page size and total_hits """ - def __init__(self, page_get, user_id, search_get=False): - self.user_id = user_id + def __init__(self, request): + self.request = request + self.page_get = False + self.params = False + self.get_params() self.page_size = self.get_page_size() - self.page_get = page_get - self.search_get = search_get self.pagination = self.first_guess() + def get_params(self): + """process url query parameters""" + query_dict = self.request.GET.copy() + self.page_get = int(query_dict.get("page", 0)) + + _ = query_dict.pop("page", False) + self.params = query_dict.urlencode() + def get_page_size(self): """get default or user modified page_size""" - key = f"{self.user_id}:page_size" + key = f"{self.request.user.id}:page_size" page_size = RedisArchivist().get_message(key)["status"] if not page_size: config = AppConfig().config @@ -108,9 +117,9 @@ class Pagination: "prev_pages": prev_pages, "current_page": page_get, "max_hits": False, + "params": self.params, } - if self.search_get: - pagination.update({"search_get": self.search_get}) + return pagination def validate(self, total_hits): diff --git a/tubearchivist/home/templates/home/base.html b/tubearchivist/home/templates/home/base.html index 83b41f93..5e539956 100644 --- a/tubearchivist/home/templates/home/base.html +++ b/tubearchivist/home/templates/home/base.html @@ -79,16 +79,16 @@