mirror of
https://git.vectorsigma.ru/public/tubearchivist.git
synced 2026-08-09 02:59:55 +00:00
Compare commits
48 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
528938942a | ||
|
|
de2371c43c | ||
|
|
1a88c85436 | ||
|
|
63be6afcfe | ||
|
|
559a4c9a58 | ||
|
|
08338f3270 | ||
|
|
0c3c1b650a | ||
|
|
45d8b7e218 | ||
|
|
5edc78a826 | ||
|
|
fd2fa628e3 | ||
|
|
ba12c2ad25 | ||
|
|
0d061d8601 | ||
|
|
0844f7b91a | ||
|
|
044b12d5a5 | ||
|
|
ce5f8b6a38 | ||
|
|
9ad42dfa38 | ||
|
|
952bc4f97c | ||
|
|
bfd51e6804 | ||
|
|
72af95acd8 | ||
|
|
3473c79d63 | ||
|
|
a4ae6ca1ca | ||
|
|
48a4827ad5 | ||
|
|
29f17680bf | ||
|
|
519c710e7d | ||
|
|
1b04c00ae7 | ||
|
|
9f12966ef7 | ||
|
|
76f49b4cf7 | ||
|
|
35a5eda36e | ||
|
|
567e9d473b | ||
|
|
e202265a5f | ||
|
|
bc84696792 | ||
|
|
f5621954fb | ||
|
|
ef75f6dd16 | ||
|
|
b53a1763c8 | ||
|
|
11780a0ff0 | ||
|
|
8d3e2f14fc | ||
|
|
7405e960e5 | ||
|
|
8c97e6786a | ||
|
|
563222a26e | ||
|
|
64ccd3830e | ||
|
|
c2a6ac6f44 | ||
|
|
0b88fd8b1f | ||
|
|
744780f4bd | ||
|
|
4d152e1e54 | ||
|
|
3500044b0d | ||
|
|
7e0abc0d20 | ||
|
|
b0fb2bbb00 | ||
|
|
4e7cba70d5 |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -3,6 +3,3 @@ __pycache__
|
||||
|
||||
# django testing db
|
||||
db.sqlite3
|
||||
|
||||
# frontend fonts
|
||||
*ttf.woff
|
||||
|
||||
@@ -27,6 +27,7 @@ This is my setup I have landed on, YMMV:
|
||||
./deploy.sh test
|
||||
```
|
||||
- The command above will also copy the file `tubarchivist/testing.sh` into the working folder of the container. Running this script will install additional debugging tools I regularly use in testing.
|
||||
- The `test` argument takes another optional argument to build for a specific architecture valid options are: `amd64`, `arm64` and `multi`, default is `amd64`.
|
||||
- This `deploy.sh` file is not meant to be universally usable for every possible environment but could serve as an idea on how to automatically rebuild containers to test changes - customize to your liking.
|
||||
|
||||
## Implementing a new feature
|
||||
|
||||
24
Dockerfile
24
Dockerfile
@@ -1,6 +1,7 @@
|
||||
# build the tube archivist image from default python slim image
|
||||
|
||||
FROM python:3.9.7-slim-bullseye
|
||||
ARG TARGETPLATFORM
|
||||
|
||||
ENV PYTHONUNBUFFERED 1
|
||||
|
||||
@@ -10,16 +11,19 @@ RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recomm
|
||||
nginx \
|
||||
curl && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# get newest patched ffmpeg and ffprobe builds
|
||||
RUN curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
|
||||
| grep browser_download_url \
|
||||
| grep linux64 \
|
||||
| grep ffmpeg-n \
|
||||
| cut -d '"' -f 4 \
|
||||
| xargs curl -L --output ffmpeg.tar.xz && \
|
||||
tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
|
||||
tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \
|
||||
rm ffmpeg.tar.xz
|
||||
# get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64
|
||||
RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
|
||||
curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
|
||||
| grep browser_download_url \
|
||||
| grep linux64-gpl-4.4.tar.xz \
|
||||
| cut -d '"' -f 4 \
|
||||
| xargs curl -L --output ffmpeg.tar.xz && \
|
||||
tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
|
||||
tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \
|
||||
rm ffmpeg.tar.xz \
|
||||
; elif [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \
|
||||
apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \
|
||||
; fi
|
||||
|
||||
# copy config files
|
||||
COPY nginx.conf /etc/nginx/conf.d/
|
||||
|
||||
18
README.md
18
README.md
@@ -73,9 +73,15 @@ For some architectures it might be required to run Redis JSON on a nonstandard p
|
||||
|
||||
### Updating Tube Archivist
|
||||
You will see the current version number of **Tube Archivist** in the footer of the interface so you can compare it with the latest release to make sure you are running the *latest and greatest*.
|
||||
* There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. Any breaking changes will be marked in the **release notes**.
|
||||
* There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.
|
||||
* All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated when a new release of Elasticsearch is available. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned.
|
||||
|
||||
### Alternative installation instructions:
|
||||
- **arm64**: Newest Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, you can use `bbilly1/rejson`, an unofficial rebuild for arm64.
|
||||
- NOTE: This is untested, looking for feedback.
|
||||
- **Synology**: There is a [discussion thread](https://github.com/bbilly1/tubearchivist/discussions/48) with Synology installation instructions.
|
||||
- **Unraid**: There is a [template](https://github.com/pairofcrocs/unraid-templates/) and forum [thread](https://forums.unraid.net/topic/114073-support-crocs-tube-archivist/) with additional information.
|
||||
|
||||
## Potential pitfalls
|
||||
### vm.max_map_count
|
||||
**Elastic Search** in Docker requires the kernel setting of the host machine `vm.max_map_count` to be set to at least 262144.
|
||||
@@ -111,22 +117,22 @@ This should be considered as a **minimal viable product**, there is an extensive
|
||||
### Functionality
|
||||
- [ ] Access control
|
||||
- [ ] User roles
|
||||
- [ ] Delete videos and channel
|
||||
- [ ] Create playlists
|
||||
- [ ] Podcast mode to serve channel as mp3
|
||||
- [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage
|
||||
- [ ] Un-ignore videos
|
||||
- [ ] Add thumbnail embed option
|
||||
- [X] Delete videos and channel [2021-10-16]
|
||||
- [X] Add thumbnail embed option [2021-10-16]
|
||||
- [X] Un-ignore videos [2021-10-03]
|
||||
- [X] Dynamic download queue [2021-09-26]
|
||||
- [X] Backup and restore [2021-09-22]
|
||||
- [X] Scan your file system to index already downloaded videos [2021-09-14]
|
||||
|
||||
### UI
|
||||
- [ ] Create a github wiki for user documentation
|
||||
- [ ] Show similar videos on video page
|
||||
- [ ] Multi language support
|
||||
- [ ] Grid and list view for both channel and video list pages
|
||||
- [ ] Show total video downloaded vs total videos available in channel
|
||||
- [X] Grid and list view for both channel and video list pages [2021-10-03]
|
||||
- [X] Create a github wiki for user documentation [2021-10-03]
|
||||
|
||||
|
||||
## Known limitations
|
||||
|
||||
35
deploy.sh
35
deploy.sh
@@ -5,11 +5,16 @@
|
||||
# blackhole for local production
|
||||
# docker to publish
|
||||
|
||||
# create builder:
|
||||
# docker buildx create --name tubearchivist
|
||||
# docker buildx use tubearchivist
|
||||
# docker buildx inspect --bootstrap
|
||||
|
||||
set -e
|
||||
|
||||
function sync_blackhole {
|
||||
|
||||
# docker commands need sudo
|
||||
# docker commands need sudo, only build amd64
|
||||
host="blackhole.local"
|
||||
|
||||
read -sp 'Password: ' remote_pw
|
||||
@@ -23,7 +28,7 @@ function sync_blackhole {
|
||||
--exclude "db.sqlite3" \
|
||||
. -e ssh "$host":tubearchivist
|
||||
|
||||
echo "$PASS" | ssh "$host" 'sudo -S docker build -t bbilly1/tubearchivist:latest tubearchivist 2>/dev/null'
|
||||
echo "$PASS" | ssh "$host" 'sudo -S docker buildx build --platform linux/amd64 -t bbilly1/tubearchivist:latest tubearchivist --load 2>/dev/null'
|
||||
echo "$PASS" | ssh "$host" 'sudo -S docker-compose up -d 2>/dev/null'
|
||||
|
||||
}
|
||||
@@ -31,6 +36,8 @@ function sync_blackhole {
|
||||
function sync_test {
|
||||
|
||||
# docker commands don't need sudo in testing vm
|
||||
# pass argument to build for specific platform
|
||||
|
||||
host="tubearchivist.local"
|
||||
|
||||
rsync -a --progress --delete-after \
|
||||
@@ -43,7 +50,17 @@ function sync_test {
|
||||
|
||||
rsync -r --progress --delete docker-compose.yml -e ssh "$host":docker
|
||||
|
||||
ssh "$host" 'docker build -t bbilly1/tubearchivist:latest tubearchivist'
|
||||
if [[ $1 = "amd64" ]]; then
|
||||
platform="linux/amd64"
|
||||
elif [[ $1 = "arm64" ]]; then
|
||||
platform="linux/arm64"
|
||||
elif [[ $1 = "multi" ]]; then
|
||||
platform="linux/amd64,linux/arm64"
|
||||
else
|
||||
platform="linux/amd64"
|
||||
fi
|
||||
|
||||
ssh "$host" "docker buildx build --platform $platform -t bbilly1/tubearchivist:latest tubearchivist --load"
|
||||
ssh "$host" 'docker-compose -f docker/docker-compose.yml up -d'
|
||||
|
||||
ssh "$host" 'docker cp tubearchivist/tubearchivist/testing.sh tubearchivist:/app/testing.sh'
|
||||
@@ -103,7 +120,7 @@ function sync_docker {
|
||||
read -r VERSION
|
||||
|
||||
# start build
|
||||
sudo docker build -t bbilly1/tubearchivist:latest -t bbilly1/tubearchivist:"$VERSION" .
|
||||
sudo docker buildx build --platform linux/amd64,linux/arm64 -t bbilly1/tubearchivist:latest -t bbilly1/tubearchivist:"$VERSION" .
|
||||
|
||||
printf "\nlatest images:\n"
|
||||
sudo docker image ls bbilly1/tubearchivist
|
||||
@@ -112,10 +129,10 @@ function sync_docker {
|
||||
read -rn 1
|
||||
|
||||
# push to docker
|
||||
echo "pushing latest:"
|
||||
sudo docker push bbilly1/tubearchivist:latest
|
||||
echo "pushing $VERSION"
|
||||
sudo docker push bbilly1/tubearchivist:"$VERSION"
|
||||
echo "pushing multiarch latest and $VERSION:"
|
||||
sudo docker buildx build --platform linux/amd64,linux/arm64 -t bbilly1/tubearchivist:latest -t bbilly1/tubearchivist:"$VERSION" --push .
|
||||
#sudo docker push bbilly1/tubearchivist:latest
|
||||
#sudo docker push bbilly1/tubearchivist:"$VERSION"
|
||||
|
||||
# create release tag
|
||||
echo "commits since last version:"
|
||||
@@ -133,7 +150,7 @@ python version_check.py
|
||||
if [[ $1 == "blackhole" ]]; then
|
||||
sync_blackhole
|
||||
elif [[ $1 == "test" ]]; then
|
||||
sync_test
|
||||
sync_test "$2"
|
||||
elif [[ $1 == "validate" ]]; then
|
||||
validate "$2"
|
||||
elif [[ $1 == "docker" ]]; then
|
||||
|
||||
@@ -29,7 +29,7 @@ services:
|
||||
depends_on:
|
||||
- archivist-es
|
||||
archivist-es:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.15.0
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
|
||||
container_name: archivist-es
|
||||
restart: always
|
||||
environment:
|
||||
|
||||
@@ -21,3 +21,4 @@ Each channel will get a dedicated channel detail page accessible at `/channel/<c
|
||||
- If you are subscribed to the channel, an *Unsubscribe* button will show.
|
||||
- You can *Show* the channel description, that matches with the *About* tab on YouTube.
|
||||
- The **Mark as Watched** button will mark all videos of this channel as watched.
|
||||
- The button **Delete Channel** will delete the channel plus all videos of this channel, both media files and metadata.
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
# Tube Archivist Wiki
|
||||
|
||||
**WIP**: This is work in progress!
|
||||
|
||||
Welcome to the official Tube Archivist Wiki. This is an up-to-date documentation of user functionality.
|
||||
|
||||
Table of contents:
|
||||
@@ -24,6 +21,6 @@ Table of contents:
|
||||
* Hover over the playing video to show additional control options.
|
||||
|
||||
|
||||
An empty checkbox icon <img src="assets/icon-unseen.png?raw=true" alt="unseen icon" width="20px" style="margin:0 5px;"> will show for videos you haven't marked as watched. Click on it and the icon will change to a filled checkbox <img src="assets/icon-seen.png?raw=true" alt="seen icon" width="20px" style="margin:0 5px;"> indicating it as watched.
|
||||
An empty checkbox icon <img src="assets/icon-unseen.png?raw=true" alt="unseen icon" width="20px" style="margin:0 5px;"> will show for videos you haven't marked as watched. Click on it and the icon will change to a filled checkbox <img src="assets/icon-seen.png?raw=true" alt="seen icon" width="20px" style="margin:0 5px;"> indicating it as watched - click again to revert.
|
||||
|
||||
When available the <img src="assets/icon-gridview.png?raw=true" alt="gridview icon" width="20px" style="margin:0 5px;"> gridview icon will display the list in a grid, the <img src="assets/icon-listview.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> listview icon will arrange the items in a list.
|
||||
|
||||
@@ -23,6 +23,7 @@ Settings related to the download process.
|
||||
Additional settings passed to yt-dlp.
|
||||
- **Format**: This controls which streams get downloaded and is equivalent to passing `--format` to yt-dlp. Use one of the recommended one or look at the documentation of [yt-dlp](https://github.com/yt-dlp/yt-dlp#format-selection). Please note: The option `--merge-output-format mp4` is automatically passed to yt-dlp to guarantee browser compatibility.
|
||||
- **Embed Metadata**: This saves the available tags directly into the media file by passing `--embed-metadata` to yt-dlp.
|
||||
- **Embed Thumbnail**: This will save the thumbnail into the media file by passing `--embed-thumbnail` to yt-dlp.
|
||||
|
||||
|
||||
# Actions
|
||||
@@ -58,3 +59,11 @@ BE AWARE: This will **not** backup any media files, just the metadata from the E
|
||||
The restore functionality will expect the same zip file in *cache/backup* as created from the **Backup database** function. This will recreate the index from the snapshot. If there are multiple backup files in the folder, the newest one will take priority.
|
||||
|
||||
BE AWARE: This will **replace** your current index with the one from the backup file. This won't restore any media files.
|
||||
|
||||
## Rescan Filesystem
|
||||
This function will go through all your media files and looks at the whole index to try to find any issues:
|
||||
- Should the filename not match with the indexed media url, this will rename the video files correctly and update the index with the new link.
|
||||
- When you delete media files from the filesystem outside of the Tube Archivist interface, this will delete leftover metadata from the index.
|
||||
- When you have media files that are not indexed yet, this will grab the metadata from YouTube like it was a newly downloaded video. This can be useful when restoring from an older backup file with missing metadata but already downloaded mediafiles. NOTE: This only works if the media files are named in the same convention as Tube Archivist does, particularly the YouTube ID needs to be at the same index in the filename, alternatively see above for *Manual Media Files Import*.
|
||||
|
||||
BE AWARE: There is no undo.
|
||||
|
||||
@@ -6,6 +6,14 @@ from django.apps import AppConfig
|
||||
from home.src.config import AppConfig as ArchivistConfig
|
||||
from home.src.helper import RedisArchivist
|
||||
from home.src.index_management import index_check
|
||||
from home.src.thumbnails import validate_thumbnails
|
||||
|
||||
|
||||
def sync_redis_state():
|
||||
"""make sure redis gets new config.json values"""
|
||||
print("sync redis")
|
||||
config_handler = ArchivistConfig()
|
||||
config_handler.load_new_defaults()
|
||||
|
||||
|
||||
def make_folders():
|
||||
@@ -39,4 +47,6 @@ class HomeConfig(AppConfig):
|
||||
def ready(self):
|
||||
release_lock()
|
||||
index_check()
|
||||
sync_redis_state()
|
||||
make_folders()
|
||||
validate_thumbnails()
|
||||
|
||||
@@ -18,9 +18,11 @@
|
||||
"limit_speed": false,
|
||||
"sleep_interval": 3,
|
||||
"format": false,
|
||||
"add_metadata": false
|
||||
"add_metadata": false,
|
||||
"add_thumbnail": false
|
||||
},
|
||||
"application": {
|
||||
"app_root": "/app",
|
||||
"cache_dir": "/cache",
|
||||
"videos": "/youtube",
|
||||
"file_template": "%(id)s_%(title)s.mp4",
|
||||
|
||||
@@ -70,6 +70,7 @@ class PendingList:
|
||||
all_downloaded = self.get_all_downloaded()
|
||||
# loop
|
||||
bulk_list = []
|
||||
all_videos_added = []
|
||||
for video in missing_videos:
|
||||
if isinstance(video, str):
|
||||
youtube_id = video
|
||||
@@ -87,10 +88,12 @@ class PendingList:
|
||||
video["channel_indexed"] = True
|
||||
else:
|
||||
video["channel_indexed"] = False
|
||||
thumb_url = video["vid_thumb_url"]
|
||||
video["status"] = "pending"
|
||||
action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
|
||||
bulk_list.append(json.dumps(action))
|
||||
bulk_list.append(json.dumps(video))
|
||||
all_videos_added.append((youtube_id, thumb_url))
|
||||
# notify
|
||||
mess_dict = {
|
||||
"status": "pending",
|
||||
@@ -108,6 +111,8 @@ class PendingList:
|
||||
if not request.ok:
|
||||
print(request)
|
||||
|
||||
return all_videos_added
|
||||
|
||||
@staticmethod
|
||||
def get_youtube_details(youtube_id):
|
||||
"""get details from youtubedl for single pending video"""
|
||||
@@ -121,9 +126,14 @@ class PendingList:
|
||||
except youtube_dl.utils.DownloadError:
|
||||
print("failed to extract info for: " + youtube_id)
|
||||
return False
|
||||
# stop if video is streaming live now
|
||||
if vid["is_live"]:
|
||||
return False
|
||||
# parse response
|
||||
seconds = vid["duration"]
|
||||
duration_str = DurationConverter.get_str(seconds)
|
||||
if duration_str == "NA":
|
||||
print(f"skip extracting duration for: {youtube_id}")
|
||||
upload_date = vid["upload_date"]
|
||||
upload_dt = datetime.strptime(upload_date, "%Y%m%d")
|
||||
published = upload_dt.strftime("%Y-%m-%d")
|
||||
@@ -165,12 +175,11 @@ class PendingList:
|
||||
all_hits = json_data["hits"]["hits"]
|
||||
if all_hits:
|
||||
for hit in all_hits:
|
||||
youtube_id = hit["_source"]["youtube_id"]
|
||||
status = hit["_source"]["status"]
|
||||
if status == "pending":
|
||||
all_pending.append(hit["_source"])
|
||||
elif status == "ignore":
|
||||
all_ignore.append(youtube_id)
|
||||
all_ignore.append(hit["_source"])
|
||||
search_after = hit["sort"]
|
||||
# update search_after with last hit data
|
||||
data["search_after"] = search_after
|
||||
@@ -342,9 +351,9 @@ class ChannelSubscription:
|
||||
all_channels = self.get_channels()
|
||||
pending_handler = PendingList()
|
||||
all_pending, all_ignore = pending_handler.get_all_pending()
|
||||
all_pending_ids = [i["youtube_id"] for i in all_pending]
|
||||
all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
|
||||
all_downloaded = pending_handler.get_all_downloaded()
|
||||
to_ignore = all_pending_ids + all_ignore + all_downloaded
|
||||
to_ignore = all_ids + all_downloaded
|
||||
missing_videos = []
|
||||
counter = 1
|
||||
for channel in all_channels:
|
||||
@@ -390,6 +399,8 @@ class ChannelSubscription:
|
||||
print(request.text)
|
||||
# sync to videos
|
||||
channel_handler.sync_to_videos()
|
||||
if channel_handler.source == "scraped":
|
||||
channel_handler.get_channel_art()
|
||||
|
||||
|
||||
def playlist_extractor(playlist_id):
|
||||
@@ -444,6 +455,18 @@ class VideoDownloader:
|
||||
"""add pending videos to download queue"""
|
||||
all_pending, _ = PendingList().get_all_pending()
|
||||
to_add = [i["youtube_id"] for i in all_pending]
|
||||
if not to_add:
|
||||
# there is nothing pending
|
||||
print("download queue is empty")
|
||||
mess_dict = {
|
||||
"status": "downloading",
|
||||
"level": "error",
|
||||
"title": "Download queue is empty",
|
||||
"message": "",
|
||||
}
|
||||
RedisArchivist().set_message("progress:download", mess_dict)
|
||||
return
|
||||
|
||||
queue = RedisQueue("dl_queue")
|
||||
queue.add_list(to_add)
|
||||
|
||||
@@ -470,8 +493,8 @@ class VideoDownloader:
|
||||
}
|
||||
RedisArchivist().set_message("progress:download", mess_dict)
|
||||
|
||||
def dl_single_vid(self, youtube_id):
|
||||
"""download single video"""
|
||||
def build_obs(self):
|
||||
"""build obs dictionary for yt-dlp"""
|
||||
obs = {
|
||||
"default_search": "ytsearch",
|
||||
"merge_output_format": "mp4",
|
||||
@@ -482,9 +505,11 @@ class VideoDownloader:
|
||||
+ self.config["application"]["file_template"]
|
||||
),
|
||||
"progress_hooks": [self.progress_hook],
|
||||
"noprogress": True,
|
||||
"quiet": True,
|
||||
"continuedl": True,
|
||||
"retries": 3,
|
||||
"writethumbnail": False,
|
||||
}
|
||||
if self.config["downloads"]["format"]:
|
||||
obs["format"] = self.config["downloads"]["format"]
|
||||
@@ -505,15 +530,29 @@ class VideoDownloader:
|
||||
}
|
||||
)
|
||||
|
||||
if self.config["downloads"]["add_thumbnail"]:
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "EmbedThumbnail",
|
||||
"already_have_thumbnail": True,
|
||||
}
|
||||
)
|
||||
obs["writethumbnail"] = True
|
||||
|
||||
obs["postprocessors"] = postprocessors
|
||||
|
||||
return obs
|
||||
|
||||
def dl_single_vid(self, youtube_id):
|
||||
"""download single video"""
|
||||
dl_cache = self.config["application"]["cache_dir"] + "/download/"
|
||||
obs = self.build_obs()
|
||||
|
||||
# check if already in cache to continue from there
|
||||
cache_dir = self.config["application"]["cache_dir"]
|
||||
cached = os.listdir(cache_dir + "/download/")
|
||||
all_cached = ignore_filelist(cached)
|
||||
all_cached = ignore_filelist(os.listdir(dl_cache))
|
||||
for file_name in all_cached:
|
||||
if youtube_id in file_name:
|
||||
obs["outtmpl"] = cache_dir + "/download/" + file_name
|
||||
obs["outtmpl"] = os.path.join(dl_cache, file_name)
|
||||
with youtube_dl.YoutubeDL(obs) as ydl:
|
||||
try:
|
||||
ydl.download([youtube_id])
|
||||
@@ -522,33 +561,36 @@ class VideoDownloader:
|
||||
sleep(10)
|
||||
ydl.download([youtube_id])
|
||||
|
||||
if obs["writethumbnail"]:
|
||||
# webp files don't get cleaned up automatically
|
||||
all_cached = ignore_filelist(os.listdir(dl_cache))
|
||||
to_clean = [i for i in all_cached if not i.endswith(".mp4")]
|
||||
for file_name in to_clean:
|
||||
file_path = os.path.join(dl_cache, file_name)
|
||||
os.remove(file_path)
|
||||
|
||||
def move_to_archive(self, vid_dict):
|
||||
"""move downloaded video from cache to archive"""
|
||||
videos = self.config["application"]["videos"]
|
||||
channel_name = vid_dict["channel"]["channel_name"]
|
||||
channel_name_clean = clean_string(channel_name)
|
||||
media_url = vid_dict["media_url"]
|
||||
youtube_id = vid_dict["youtube_id"]
|
||||
# make archive folder
|
||||
videos = self.config["application"]["videos"]
|
||||
new_folder = os.path.join(videos, channel_name_clean)
|
||||
os.makedirs(new_folder, exist_ok=True)
|
||||
host_uid = self.config["application"]["HOST_UID"]
|
||||
host_gid = self.config["application"]["HOST_GID"]
|
||||
channel_name = clean_string(vid_dict["channel"]["channel_name"])
|
||||
# make archive folder with correct permissions
|
||||
new_folder = os.path.join(videos, channel_name)
|
||||
if not os.path.exists(new_folder):
|
||||
os.makedirs(new_folder)
|
||||
os.chown(new_folder, host_uid, host_gid)
|
||||
# find real filename
|
||||
cache_dir = self.config["application"]["cache_dir"]
|
||||
cached = os.listdir(cache_dir + "/download/")
|
||||
all_cached = ignore_filelist(cached)
|
||||
all_cached = ignore_filelist(os.listdir(cache_dir + "/download/"))
|
||||
for file_str in all_cached:
|
||||
if youtube_id in file_str:
|
||||
if vid_dict["youtube_id"] in file_str:
|
||||
old_file = file_str
|
||||
old_file_path = os.path.join(cache_dir, "download", old_file)
|
||||
new_file_path = os.path.join(videos, media_url)
|
||||
# move and fix permission
|
||||
new_file_path = os.path.join(videos, vid_dict["media_url"])
|
||||
# move media file and fix permission
|
||||
shutil.move(old_file_path, new_file_path)
|
||||
os.chown(
|
||||
new_file_path,
|
||||
self.config["application"]["HOST_UID"],
|
||||
self.config["application"]["HOST_GID"],
|
||||
)
|
||||
os.chown(new_file_path, host_uid, host_gid)
|
||||
|
||||
def delete_from_pending(self, youtube_id):
|
||||
"""delete downloaded video from pending index if its there"""
|
||||
|
||||
@@ -237,6 +237,10 @@ class DurationConverter:
|
||||
@staticmethod
|
||||
def get_str(duration_sec):
|
||||
"""takes duration in sec and returns clean string"""
|
||||
if not duration_sec:
|
||||
# failed to extract
|
||||
return "NA"
|
||||
|
||||
hours = duration_sec // 3600
|
||||
minutes = (duration_sec - (hours * 3600)) // 60
|
||||
secs = duration_sec - (hours * 3600) - (minutes * 60)
|
||||
|
||||
@@ -16,6 +16,7 @@ import yt_dlp as youtube_dl
|
||||
from bs4 import BeautifulSoup
|
||||
from home.src.config import AppConfig
|
||||
from home.src.helper import DurationConverter, clean_string, process_url_list
|
||||
from home.src.thumbnails import ThumbManager
|
||||
|
||||
|
||||
class YoutubeChannel:
|
||||
@@ -24,6 +25,7 @@ class YoutubeChannel:
|
||||
CONFIG = AppConfig().config
|
||||
ES_URL = CONFIG["application"]["es_url"]
|
||||
CACHE_DIR = CONFIG["application"]["cache_dir"]
|
||||
VIDEOS = CONFIG["application"]["videos"]
|
||||
|
||||
def __init__(self, channel_id):
|
||||
self.channel_id = channel_id
|
||||
@@ -152,6 +154,15 @@ class YoutubeChannel:
|
||||
|
||||
return meta_channel_dict
|
||||
|
||||
def get_channel_art(self):
|
||||
"""download channel art for new channels"""
|
||||
channel_id = self.channel_id
|
||||
channel_thumb = self.channel_dict["channel_thumb_url"]
|
||||
channel_banner = self.channel_dict["channel_banner_url"]
|
||||
ThumbManager().download_chan(
|
||||
[(channel_id, channel_thumb, channel_banner)]
|
||||
)
|
||||
|
||||
def upload_to_es(self):
|
||||
"""upload channel data to elastic search"""
|
||||
url = f"{self.ES_URL}/ta_channel/_doc/{self.channel_id}"
|
||||
@@ -160,16 +171,6 @@ class YoutubeChannel:
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
|
||||
def clear_cache(self):
|
||||
"""delete banner and thumb from cache if available"""
|
||||
channel_cache = os.path.join(self.CACHE_DIR, "channels")
|
||||
thumb = os.path.join(channel_cache, self.channel_id + "_thumb.jpg")
|
||||
banner = os.path.join(channel_cache, self.channel_id + "_banner.jpg")
|
||||
if os.path.exists(thumb):
|
||||
os.remove(thumb)
|
||||
if os.path.exists(banner):
|
||||
os.remove(banner)
|
||||
|
||||
def sync_to_videos(self):
|
||||
"""sync new channel_dict to all videos of channel"""
|
||||
headers = {"Content-type": "application/json"}
|
||||
@@ -193,17 +194,45 @@ class YoutubeChannel:
|
||||
if not request.ok:
|
||||
print(request.text)
|
||||
|
||||
def get_total_hits(self):
|
||||
"""get total channels indexed"""
|
||||
def get_folder_path(self):
|
||||
"""get folder where media files get stored"""
|
||||
channel_name = self.channel_dict["channel_name"]
|
||||
folder_name = clean_string(channel_name)
|
||||
folder_path = os.path.join(self.VIDEOS, folder_name)
|
||||
return folder_path
|
||||
|
||||
def delete_es_videos(self):
|
||||
"""delete all channel documents from elasticsearch"""
|
||||
headers = {"Content-type": "application/json"}
|
||||
data = {"query": {"match_all": {}}}
|
||||
data = {
|
||||
"query": {
|
||||
"term": {"channel.channel_id": {"value": self.channel_id}}
|
||||
}
|
||||
}
|
||||
payload = json.dumps(data)
|
||||
url = f"{self.ES_URL}/ta_channel/_search?filter_path=hits.total"
|
||||
request = requests.post(url, data=payload, headers=headers)
|
||||
if not request.ok:
|
||||
print(request.text)
|
||||
total_hits = json.loads(request.text)["hits"]["total"]["value"]
|
||||
return total_hits
|
||||
url = self.ES_URL + "/ta_video/_delete_by_query"
|
||||
response = requests.post(url, data=payload, headers=headers)
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
|
||||
def delete_channel(self):
|
||||
"""delete channel and all videos"""
|
||||
print(f"deleting {self.channel_id} and all matching media files")
|
||||
folder_path = self.get_folder_path()
|
||||
print("delete all media files")
|
||||
all_videos = os.listdir(folder_path)
|
||||
for video in all_videos:
|
||||
video_path = os.path.join(folder_path, video)
|
||||
os.remove(video_path)
|
||||
os.rmdir(folder_path)
|
||||
ThumbManager().delete_chan_thumb(self.channel_id)
|
||||
|
||||
print("delete indexed videos")
|
||||
self.delete_es_videos()
|
||||
url = self.ES_URL + "/ta_channel/_doc/" + self.channel_id
|
||||
response = requests.delete(url)
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
|
||||
|
||||
class YoutubeVideo:
|
||||
@@ -217,11 +246,12 @@ class YoutubeVideo:
|
||||
def __init__(self, youtube_id):
|
||||
self.youtube_id = youtube_id
|
||||
self.channel_id = None
|
||||
self.vid_dict = self.get_wrapper()
|
||||
self.vid_dict = None
|
||||
|
||||
def get_wrapper(self):
|
||||
def get_vid_dict(self):
|
||||
"""wrapper to loop around youtube_dl to retry on failure"""
|
||||
print(f"get video data for {self.youtube_id}")
|
||||
vid_dict = False
|
||||
for i in range(3):
|
||||
try:
|
||||
vid_dict = self.get_youtubedl_vid_data()
|
||||
@@ -232,7 +262,7 @@ class YoutubeVideo:
|
||||
else:
|
||||
break
|
||||
|
||||
return vid_dict
|
||||
self.vid_dict = vid_dict
|
||||
|
||||
def get_youtubedl_vid_data(self):
|
||||
"""parse youtubedl extract info"""
|
||||
@@ -343,13 +373,6 @@ class YoutubeVideo:
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
|
||||
def delete_cache(self):
|
||||
"""delete thumbnail from cache if exist"""
|
||||
video_cache = os.path.join(self.CACHE_DIR, "videos")
|
||||
thumb = os.path.join(video_cache, self.youtube_id + ".jpg")
|
||||
if os.path.exists(thumb):
|
||||
os.remove(thumb)
|
||||
|
||||
def deactivate(self):
|
||||
"""deactivate document on extractor error"""
|
||||
youtube_id = self.youtube_id
|
||||
@@ -362,6 +385,22 @@ class YoutubeVideo:
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
|
||||
def delete_media_file(self):
|
||||
"""delete video file, meta data, thumbnails"""
|
||||
# delete media file
|
||||
es_vid_dict = self.get_es_data()
|
||||
media_url = es_vid_dict["_source"]["media_url"]
|
||||
print(f"delete {media_url} from file system")
|
||||
to_delete = os.path.join(self.VIDEOS, media_url)
|
||||
os.remove(to_delete)
|
||||
# delete from index
|
||||
url = f"{self.ES_URL}/ta_video/_doc/{self.youtube_id}"
|
||||
response = requests.delete(url)
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
# delete thumbs from cache
|
||||
ThumbManager().delete_vid_thumb(self.youtube_id)
|
||||
|
||||
|
||||
class WatchState:
|
||||
"""handle watched checkbox for videos and channels"""
|
||||
@@ -384,6 +423,16 @@ class WatchState:
|
||||
|
||||
print(f"marked {self.youtube_id} as watched")
|
||||
|
||||
def mark_as_unwatched(self):
|
||||
"""revert watched state to false"""
|
||||
url_type = self.dedect_type()
|
||||
if url_type == "video":
|
||||
self.mark_vid_watched(revert=True)
|
||||
elif url_type == "channel":
|
||||
self.mark_channel_watched(revert=True)
|
||||
|
||||
print(f"revert {self.youtube_id} as unwatched")
|
||||
|
||||
def dedect_type(self):
|
||||
"""find youtube id type"""
|
||||
url_process = process_url_list([self.youtube_id])
|
||||
@@ -391,18 +440,21 @@ class WatchState:
|
||||
|
||||
return url_type
|
||||
|
||||
def mark_vid_watched(self):
|
||||
def mark_vid_watched(self, revert=False):
|
||||
"""change watched status of single video"""
|
||||
url = self.ES_URL + "/ta_video/_update/" + self.youtube_id
|
||||
data = {
|
||||
"doc": {"player": {"watched": True, "watched_date": self.stamp}}
|
||||
}
|
||||
if revert:
|
||||
data["doc"]["player"]["watched"] = False
|
||||
|
||||
payload = json.dumps(data)
|
||||
request = requests.post(url, data=payload, headers=self.HEADERS)
|
||||
if not request.ok:
|
||||
print(request.text)
|
||||
|
||||
def mark_channel_watched(self):
|
||||
def mark_channel_watched(self, revert=False):
|
||||
"""change watched status of every video in channel"""
|
||||
es_url = self.ES_URL
|
||||
headers = self.HEADERS
|
||||
@@ -415,6 +467,9 @@ class WatchState:
|
||||
{"set": {"field": "player.watched_date", "value": self.stamp}},
|
||||
],
|
||||
}
|
||||
if revert:
|
||||
data["processors"][0]["set"]["value"] = False
|
||||
|
||||
payload = json.dumps(data)
|
||||
url = f"{es_url}/_ingest/pipeline/{youtube_id}"
|
||||
request = requests.put(url, data=payload, headers=headers)
|
||||
@@ -438,6 +493,7 @@ class WatchState:
|
||||
def index_new_video(youtube_id, missing_vid=False):
|
||||
"""combine video and channel classes for new video index"""
|
||||
vid_handler = YoutubeVideo(youtube_id)
|
||||
vid_handler.get_vid_dict()
|
||||
if not vid_handler.vid_dict:
|
||||
raise ValueError("failed to get metadata for " + youtube_id)
|
||||
|
||||
@@ -452,6 +508,7 @@ def index_new_video(youtube_id, missing_vid=False):
|
||||
if channel_handler.source == "scraped":
|
||||
channel_handler.channel_dict["channel_subscribed"] = False
|
||||
channel_handler.upload_to_es()
|
||||
channel_handler.get_channel_art()
|
||||
# upload video to es
|
||||
vid_handler.upload_to_es()
|
||||
# return vid_dict for further processing
|
||||
|
||||
@@ -24,6 +24,7 @@ from home.src.helper import (
|
||||
ignore_filelist,
|
||||
)
|
||||
from home.src.index import YoutubeChannel, YoutubeVideo, index_new_video
|
||||
from home.src.thumbnails import ThumbManager
|
||||
|
||||
|
||||
class Reindex:
|
||||
@@ -144,6 +145,7 @@ class Reindex:
|
||||
def reindex_single_video(youtube_id):
|
||||
"""refresh data for single video"""
|
||||
vid_handler = YoutubeVideo(youtube_id)
|
||||
vid_handler.get_vid_dict()
|
||||
if not vid_handler.vid_dict:
|
||||
# stop if deactivated
|
||||
vid_handler.deactivate()
|
||||
@@ -161,7 +163,10 @@ class Reindex:
|
||||
vid_handler.vid_dict["channel"] = channel_dict
|
||||
# update
|
||||
vid_handler.upload_to_es()
|
||||
vid_handler.delete_cache()
|
||||
thumb_handler = ThumbManager()
|
||||
thumb_handler.delete_vid_thumb(youtube_id)
|
||||
to_download = (youtube_id, vid_handler.vid_dict["vid_thumb_url"])
|
||||
thumb_handler.download_vid([to_download], notify=False)
|
||||
|
||||
@staticmethod
|
||||
def reindex_single_channel(channel_id):
|
||||
@@ -172,9 +177,15 @@ class Reindex:
|
||||
scrape=True
|
||||
)
|
||||
channel_handler.channel_dict["channel_subscribed"] = subscribed
|
||||
# update
|
||||
channel_handler.upload_to_es()
|
||||
channel_handler.sync_to_videos()
|
||||
channel_handler.clear_cache()
|
||||
thumb_handler = ThumbManager()
|
||||
thumb_handler.delete_chan_thumb(channel_id)
|
||||
channel_thumb = channel_handler.channel_dict["channel_thumb_url"]
|
||||
channel_banner = channel_handler.channel_dict["channel_banner_url"]
|
||||
to_download = (channel_id, channel_thumb, channel_banner)
|
||||
thumb_handler.download_chan([to_download])
|
||||
|
||||
def reindex(self):
|
||||
"""reindex what's needed"""
|
||||
@@ -297,6 +308,7 @@ class FilesystemScanner:
|
||||
"""rename media files as identified by find_bad_media_url"""
|
||||
for bad_filename in self.to_rename:
|
||||
channel, filename, expected_filename = bad_filename
|
||||
print(f"renaming [{filename}] to [{expected_filename}]")
|
||||
old_path = os.path.join(self.VIDEOS, channel, filename)
|
||||
new_path = os.path.join(self.VIDEOS, channel, expected_filename)
|
||||
os.rename(old_path, new_path)
|
||||
@@ -306,6 +318,7 @@ class FilesystemScanner:
|
||||
bulk_list = []
|
||||
for video_mismatch in self.mismatch:
|
||||
youtube_id, media_url = video_mismatch
|
||||
print(f"{youtube_id}: fixing media url {media_url}")
|
||||
action = {"update": {"_id": youtube_id, "_index": "ta_video"}}
|
||||
source = {"doc": {"media_url": media_url}}
|
||||
bulk_list.append(json.dumps(action))
|
||||
@@ -323,7 +336,8 @@ class FilesystemScanner:
|
||||
def delete_from_index(self):
|
||||
"""find indexed but deleted mediafile"""
|
||||
for indexed in self.to_delete:
|
||||
youtube_id, _ = indexed
|
||||
youtube_id = indexed[0]
|
||||
print(f"deleting {youtube_id} from index")
|
||||
url = self.ES_URL + "/ta_video/_doc/" + youtube_id
|
||||
request = requests.delete(url)
|
||||
if not request.ok:
|
||||
@@ -399,6 +413,8 @@ class ManualImport:
|
||||
def process_import(self):
|
||||
"""go through identified media files"""
|
||||
|
||||
all_videos_added = []
|
||||
|
||||
for media_file in self.identified:
|
||||
json_file = media_file["json_file"]
|
||||
video_file = media_file["video_file"]
|
||||
@@ -411,6 +427,9 @@ class ManualImport:
|
||||
# identify and archive
|
||||
vid_dict = index_new_video(youtube_id)
|
||||
VideoDownloader([youtube_id]).move_to_archive(vid_dict)
|
||||
youtube_id = vid_dict["youtube_id"]
|
||||
thumb_url = vid_dict["vid_thumb_url"]
|
||||
all_videos_added.append((youtube_id, thumb_url))
|
||||
|
||||
# cleanup
|
||||
if os.path.exists(video_path):
|
||||
@@ -419,6 +438,8 @@ class ManualImport:
|
||||
json_path = os.path.join(self.CACHE_DIR, "import", json_file)
|
||||
os.remove(json_path)
|
||||
|
||||
return all_videos_added
|
||||
|
||||
def move_to_cache(self, video_path, youtube_id):
|
||||
"""move identified video file to cache, convert to mp4"""
|
||||
file_name = os.path.split(video_path)[-1]
|
||||
@@ -456,12 +477,16 @@ def scan_filesystem():
|
||||
filesystem_handler = FilesystemScanner()
|
||||
filesystem_handler.list_comarison()
|
||||
if filesystem_handler.to_rename:
|
||||
print("renaming files")
|
||||
filesystem_handler.rename_files()
|
||||
if filesystem_handler.mismatch:
|
||||
print("fixing media urls in index")
|
||||
filesystem_handler.send_mismatch_bulk()
|
||||
if filesystem_handler.to_delete:
|
||||
print("delete metadata from index")
|
||||
filesystem_handler.delete_from_index()
|
||||
if filesystem_handler.to_index:
|
||||
print("index new videos")
|
||||
for missing_vid in filesystem_handler.to_index:
|
||||
youtube_id = missing_vid[2]
|
||||
index_new_video(youtube_id, missing_vid=missing_vid)
|
||||
|
||||
@@ -7,14 +7,12 @@ Functionality:
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import urllib.parse
|
||||
from datetime import datetime
|
||||
|
||||
import requests
|
||||
from home.src.config import AppConfig
|
||||
from home.src.helper import ignore_filelist
|
||||
from PIL import Image
|
||||
from home.src.thumbnails import ThumbManager
|
||||
|
||||
|
||||
class SearchHandler:
|
||||
@@ -23,11 +21,10 @@ class SearchHandler:
|
||||
CONFIG = AppConfig().config
|
||||
CACHE_DIR = CONFIG["application"]["cache_dir"]
|
||||
|
||||
def __init__(self, url, data, cache=True):
|
||||
def __init__(self, url, data):
|
||||
self.max_hits = None
|
||||
self.url = url
|
||||
self.data = data
|
||||
self.cache = cache
|
||||
|
||||
def get_data(self):
|
||||
"""get the data"""
|
||||
@@ -61,10 +58,6 @@ class SearchHandler:
|
||||
channel_dict = self.channel_cache_link(hit)
|
||||
if channel_dict not in all_channels:
|
||||
all_channels.append(channel_dict)
|
||||
if self.cache:
|
||||
# validate cache
|
||||
self.cache_dl_vids(all_videos)
|
||||
self.cache_dl_chan(all_channels)
|
||||
|
||||
return return_value
|
||||
|
||||
@@ -103,50 +96,6 @@ class SearchHandler:
|
||||
}
|
||||
return channel_dict
|
||||
|
||||
def cache_dl_vids(self, all_videos):
|
||||
"""video thumbs links for cache"""
|
||||
vid_cache = os.path.join(self.CACHE_DIR, "videos")
|
||||
vid_cached = os.listdir(vid_cache)
|
||||
all_vid_cached = ignore_filelist(vid_cached)
|
||||
# videos
|
||||
for video_dict in all_videos:
|
||||
youtube_id = video_dict["youtube_id"]
|
||||
if not youtube_id + ".jpg" in all_vid_cached:
|
||||
cache_path = os.path.join(vid_cache, youtube_id + ".jpg")
|
||||
thumb_url = video_dict["vid_thumb"]
|
||||
img_raw = requests.get(thumb_url, stream=True).raw
|
||||
img = Image.open(img_raw)
|
||||
width, height = img.size
|
||||
if not width / height == 16 / 9:
|
||||
new_height = width / 16 * 9
|
||||
offset = (height - new_height) / 2
|
||||
img = img.crop((0, offset, width, height - offset))
|
||||
img.convert("RGB").save(cache_path)
|
||||
|
||||
def cache_dl_chan(self, all_channels):
|
||||
"""download channel thumbs"""
|
||||
chan_cache = os.path.join(self.CACHE_DIR, "channels")
|
||||
chan_cached = os.listdir(chan_cache)
|
||||
all_chan_cached = ignore_filelist(chan_cached)
|
||||
for channel_dict in all_channels:
|
||||
channel_id_cache = channel_dict["channel_id"]
|
||||
channel_banner_url = channel_dict["chan_banner"]
|
||||
channel_banner = channel_id_cache + "_banner.jpg"
|
||||
channel_thumb_url = channel_dict["chan_thumb"]
|
||||
channel_thumb = channel_id_cache + "_thumb.jpg"
|
||||
# thumb
|
||||
if channel_thumb_url and channel_thumb not in all_chan_cached:
|
||||
cache_path = os.path.join(chan_cache, channel_thumb)
|
||||
img_raw = requests.get(channel_thumb_url, stream=True).content
|
||||
with open(cache_path, "wb") as f:
|
||||
f.write(img_raw)
|
||||
# banner
|
||||
if channel_banner_url and channel_banner not in all_chan_cached:
|
||||
cache_path = os.path.join(chan_cache, channel_banner)
|
||||
img_raw = requests.get(channel_banner_url, stream=True).content
|
||||
with open(cache_path, "wb") as f:
|
||||
f.write(img_raw)
|
||||
|
||||
@staticmethod
|
||||
def hit_cleanup(hit):
|
||||
"""clean up and parse data from a single hit"""
|
||||
@@ -168,6 +117,11 @@ class SearchHandler:
|
||||
date_str = datetime.strftime(date_refresh, "%d %b, %Y")
|
||||
hit["source"]["vid_last_refresh"] = date_str
|
||||
|
||||
if "vid_thumb_url" in hit_keys:
|
||||
youtube_id = hit["source"]["youtube_id"]
|
||||
thumb_path = ThumbManager().vid_thumb_path(youtube_id)
|
||||
hit["source"]["vid_thumb_url"] = thumb_path
|
||||
|
||||
if "channel_last_refresh" in hit_keys:
|
||||
refreshed = hit["source"]["channel_last_refresh"]
|
||||
date_refresh = datetime.fromtimestamp(refreshed)
|
||||
@@ -208,7 +162,7 @@ class SearchForm:
|
||||
}
|
||||
},
|
||||
}
|
||||
look_up = SearchHandler(url, data, cache=False)
|
||||
look_up = SearchHandler(url, data)
|
||||
search_results = look_up.get_data()
|
||||
return {"results": search_results}
|
||||
|
||||
|
||||
243
tubearchivist/home/src/thumbnails.py
Normal file
243
tubearchivist/home/src/thumbnails.py
Normal file
@@ -0,0 +1,243 @@
|
||||
"""
|
||||
functionality:
|
||||
- handle download and caching for thumbnails
|
||||
"""
|
||||
|
||||
import os
|
||||
from collections import Counter
|
||||
from time import sleep
|
||||
|
||||
import home.src.download as download
|
||||
import requests
|
||||
from home.src.config import AppConfig
|
||||
from home.src.helper import RedisArchivist, ignore_filelist
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class ThumbManager:
|
||||
"""handle thumbnails related functions"""
|
||||
|
||||
CONFIG = AppConfig().config
|
||||
CACHE_DIR = CONFIG["application"]["cache_dir"]
|
||||
VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
|
||||
CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
|
||||
|
||||
def get_all_thumbs(self):
|
||||
"""get all video artwork already downloaded"""
|
||||
all_thumb_folders = ignore_filelist(os.listdir(self.VIDEO_DIR))
|
||||
all_thumbs = []
|
||||
for folder in all_thumb_folders:
|
||||
folder_path = os.path.join(self.VIDEO_DIR, folder)
|
||||
if os.path.isfile(folder_path):
|
||||
self.update_path(folder)
|
||||
all_thumbs.append(folder_path)
|
||||
continue
|
||||
# raise exemption here in a future version
|
||||
# raise FileExistsError("video cache dir has files inside")
|
||||
|
||||
all_folder_thumbs = ignore_filelist(os.listdir(folder_path))
|
||||
all_thumbs.extend(all_folder_thumbs)
|
||||
|
||||
return all_thumbs
|
||||
|
||||
def update_path(self, file_name):
|
||||
"""reorganize thumbnails into folders as update path from v0.0.5"""
|
||||
folder_name = file_name[0].lower()
|
||||
folder_path = os.path.join(self.VIDEO_DIR, folder_name)
|
||||
old_file = os.path.join(self.VIDEO_DIR, file_name)
|
||||
new_file = os.path.join(folder_path, file_name)
|
||||
os.makedirs(folder_path, exist_ok=True)
|
||||
os.rename(old_file, new_file)
|
||||
|
||||
def get_needed_thumbs(self, missing_only=False):
|
||||
"""get a list of all missing thumbnails"""
|
||||
all_thumbs = self.get_all_thumbs()
|
||||
all_indexed = download.PendingList().get_all_indexed()
|
||||
all_in_queue, all_ignored = download.PendingList().get_all_pending()
|
||||
|
||||
needed_thumbs = []
|
||||
for video in all_indexed:
|
||||
youtube_id = video["_source"]["youtube_id"]
|
||||
thumb_url = video["_source"]["vid_thumb_url"]
|
||||
if missing_only:
|
||||
if youtube_id + ".jpg" not in all_thumbs:
|
||||
needed_thumbs.append((youtube_id, thumb_url))
|
||||
else:
|
||||
needed_thumbs.append((youtube_id, thumb_url))
|
||||
|
||||
for video in all_in_queue + all_ignored:
|
||||
youtube_id = video["youtube_id"]
|
||||
thumb_url = video["vid_thumb_url"]
|
||||
if missing_only:
|
||||
if youtube_id + ".jpg" not in all_thumbs:
|
||||
needed_thumbs.append((youtube_id, thumb_url))
|
||||
else:
|
||||
needed_thumbs.append((youtube_id, thumb_url))
|
||||
|
||||
return needed_thumbs
|
||||
|
||||
def get_missing_channels(self):
|
||||
"""get all channel artwork"""
|
||||
all_channel_art = os.listdir(self.CHANNEL_DIR)
|
||||
files = [i[0:24] for i in all_channel_art]
|
||||
cached_channel_ids = [k for (k, v) in Counter(files).items() if v > 1]
|
||||
channels = download.ChannelSubscription().get_channels(
|
||||
subscribed_only=False
|
||||
)
|
||||
|
||||
missing_channels = []
|
||||
for channel in channels:
|
||||
channel_id = channel["channel_id"]
|
||||
if channel_id not in cached_channel_ids:
|
||||
channel_banner = channel["channel_banner_url"]
|
||||
channel_thumb = channel["channel_thumb_url"]
|
||||
missing_channels.append(
|
||||
(channel_id, channel_thumb, channel_banner)
|
||||
)
|
||||
|
||||
return missing_channels
|
||||
|
||||
def get_raw_img(self, img_url, thumb_type):
|
||||
"""get raw image from youtube and handle 404"""
|
||||
try:
|
||||
app_root = self.CONFIG["application"]["app_root"]
|
||||
except KeyError:
|
||||
# lazy keyerror fix to not have to deal with a strange startup
|
||||
# racing contition between the threads in HomeConfig.ready()
|
||||
app_root = "/app"
|
||||
default_map = {
|
||||
"video": os.path.join(
|
||||
app_root, "static/img/default-video-thumb.jpg"
|
||||
),
|
||||
"icon": os.path.join(
|
||||
app_root, "static/img/default-channel-icon.jpg"
|
||||
),
|
||||
"banner": os.path.join(
|
||||
app_root, "static/img/default-channel-banner.jpg"
|
||||
),
|
||||
}
|
||||
if img_url:
|
||||
response = requests.get(img_url, stream=True)
|
||||
if not response.ok and not response.status_code == 404:
|
||||
print("retry thumbnail download for " + img_url)
|
||||
sleep(5)
|
||||
response = requests.get(img_url, stream=True)
|
||||
else:
|
||||
response = False
|
||||
if not response or response.status_code == 404:
|
||||
# use default
|
||||
img_raw = Image.open(default_map[thumb_type])
|
||||
else:
|
||||
# use response
|
||||
img_obj = response.raw
|
||||
img_raw = Image.open(img_obj)
|
||||
|
||||
return img_raw
|
||||
|
||||
def download_vid(self, missing_thumbs, notify=True):
|
||||
"""download all missing thumbnails from list"""
|
||||
total = len(missing_thumbs)
|
||||
print(f"downloading {total} thumbnails")
|
||||
# videos
|
||||
# counter as update path from v0.0.5, remove in future version
|
||||
counter = 0
|
||||
for youtube_id, thumb_url in missing_thumbs:
|
||||
folder_name = youtube_id[0].lower()
|
||||
folder_path = os.path.join(self.VIDEO_DIR, folder_name)
|
||||
thumb_path_part = self.vid_thumb_path(youtube_id)
|
||||
thumb_path = os.path.join(self.CACHE_DIR, thumb_path_part)
|
||||
|
||||
os.makedirs(folder_path, exist_ok=True)
|
||||
img_raw = self.get_raw_img(thumb_url, "video")
|
||||
|
||||
width, height = img_raw.size
|
||||
if not width / height == 16 / 9:
|
||||
new_height = width / 16 * 9
|
||||
offset = (height - new_height) / 2
|
||||
img_raw = img_raw.crop((0, offset, width, height - offset))
|
||||
|
||||
img_raw.convert("RGB").save(thumb_path)
|
||||
|
||||
if notify:
|
||||
mess_dict = {
|
||||
"status": "pending",
|
||||
"level": "info",
|
||||
"title": "Adding to download queue.",
|
||||
"message": "Downloading Thumbnails...",
|
||||
}
|
||||
RedisArchivist().set_message("progress:download", mess_dict)
|
||||
|
||||
if counter % 50 == 0 and counter != 0:
|
||||
print(f"thumbnail progress: {counter}/{total}")
|
||||
counter = counter + 1
|
||||
|
||||
def download_chan(self, missing_channels):
|
||||
"""download needed artwork for channels"""
|
||||
print(f"downloading {len(missing_channels)} channel artwork")
|
||||
for channel in missing_channels:
|
||||
channel_id, channel_thumb, channel_banner = channel
|
||||
|
||||
thumb_path = os.path.join(
|
||||
self.CHANNEL_DIR, channel_id + "_thumb.jpg"
|
||||
)
|
||||
img_raw = self.get_raw_img(channel_thumb, "icon")
|
||||
img_raw.convert("RGB").save(thumb_path)
|
||||
|
||||
banner_path = os.path.join(
|
||||
self.CHANNEL_DIR, channel_id + "_banner.jpg"
|
||||
)
|
||||
img_raw = self.get_raw_img(channel_banner, "banner")
|
||||
img_raw.convert("RGB").save(banner_path)
|
||||
|
||||
mess_dict = {
|
||||
"status": "pending",
|
||||
"level": "info",
|
||||
"title": "Adding to download queue.",
|
||||
"message": "Downloading Channel Art...",
|
||||
}
|
||||
RedisArchivist().set_message("progress:download", mess_dict)
|
||||
|
||||
@staticmethod
|
||||
def vid_thumb_path(youtube_id):
|
||||
"""build expected path for video thumbnail from youtube_id"""
|
||||
folder_name = youtube_id[0].lower()
|
||||
folder_path = os.path.join("videos", folder_name)
|
||||
thumb_path = os.path.join(folder_path, youtube_id + ".jpg")
|
||||
return thumb_path
|
||||
|
||||
def delete_vid_thumb(self, youtube_id):
|
||||
"""delete video thumbnail if exists"""
|
||||
thumb_path = self.vid_thumb_path(youtube_id)
|
||||
to_delete = os.path.join(self.CACHE_DIR, thumb_path)
|
||||
if os.path.exists(to_delete):
|
||||
os.remove(to_delete)
|
||||
|
||||
def delete_chan_thumb(self, channel_id):
|
||||
"""delete all artwork of channel"""
|
||||
thumb = os.path.join(self.CHANNEL_DIR, channel_id + "_thumb.jpg")
|
||||
banner = os.path.join(self.CHANNEL_DIR, channel_id + "_banner.jpg")
|
||||
if os.path.exists(thumb):
|
||||
os.remove(thumb)
|
||||
if os.path.exists(banner):
|
||||
os.remove(banner)
|
||||
|
||||
def cleanup_downloaded(self):
|
||||
"""find downloaded thumbnails without video indexed"""
|
||||
all_thumbs = self.get_all_thumbs()
|
||||
all_indexed = self.get_needed_thumbs()
|
||||
all_needed_thumbs = [i[0] + ".jpg" for i in all_indexed]
|
||||
for thumb in all_thumbs:
|
||||
if thumb not in all_needed_thumbs:
|
||||
# cleanup
|
||||
youtube_id = thumb.rstrip(".jpg")
|
||||
self.delete_vid_thumb(youtube_id)
|
||||
|
||||
|
||||
def validate_thumbnails():
|
||||
"""check if all thumbnails are there and organized correctly"""
|
||||
handler = ThumbManager()
|
||||
thumbs_to_download = handler.get_needed_thumbs(missing_only=True)
|
||||
handler.download_vid(thumbs_to_download)
|
||||
missing_channels = handler.get_missing_channels()
|
||||
handler.download_chan(missing_channels)
|
||||
handler.cleanup_downloaded()
|
||||
@@ -11,7 +11,12 @@ from home.src.config import AppConfig
|
||||
from home.src.download import ChannelSubscription, PendingList, VideoDownloader
|
||||
from home.src.helper import RedisArchivist, RedisQueue
|
||||
from home.src.index_management import backup_all_indexes, restore_from_backup
|
||||
from home.src.reindex import ManualImport, reindex_old_documents
|
||||
from home.src.reindex import (
|
||||
ManualImport,
|
||||
reindex_old_documents,
|
||||
scan_filesystem,
|
||||
)
|
||||
from home.src.thumbnails import ThumbManager
|
||||
|
||||
CONFIG = AppConfig().config
|
||||
REDIS_HOST = os.environ.get("REDIS_HOST")
|
||||
@@ -33,7 +38,8 @@ def update_subscribed():
|
||||
missing_videos = channel_handler.find_missing()
|
||||
if missing_videos:
|
||||
pending_handler = PendingList()
|
||||
pending_handler.add_to_pending(missing_videos)
|
||||
all_videos_added = pending_handler.add_to_pending(missing_videos)
|
||||
ThumbManager().download_vid(all_videos_added)
|
||||
# check if reindex is needed
|
||||
check_reindex.delay()
|
||||
|
||||
@@ -41,7 +47,6 @@ def update_subscribed():
|
||||
@shared_task
|
||||
def download_pending():
|
||||
"""download latest pending videos"""
|
||||
|
||||
have_lock = False
|
||||
my_lock = RedisArchivist().get_lock("downloading")
|
||||
|
||||
@@ -62,11 +67,9 @@ def download_pending():
|
||||
@shared_task
|
||||
def download_single(youtube_id):
|
||||
"""start download single video now"""
|
||||
|
||||
queue = RedisQueue("dl_queue")
|
||||
queue.add_priority(youtube_id)
|
||||
print("Added to queue with priority: " + youtube_id)
|
||||
|
||||
# start queue if needed
|
||||
have_lock = False
|
||||
my_lock = RedisArchivist().get_lock("downloading")
|
||||
@@ -89,7 +92,8 @@ def extrac_dl(youtube_ids):
|
||||
"""parse list passed and add to pending"""
|
||||
pending_handler = PendingList()
|
||||
missing_videos = pending_handler.parse_url_list(youtube_ids)
|
||||
pending_handler.add_to_pending(missing_videos)
|
||||
all_videos_added = pending_handler.add_to_pending(missing_videos)
|
||||
ThumbManager().download_vid(all_videos_added)
|
||||
|
||||
|
||||
@shared_task
|
||||
@@ -101,7 +105,6 @@ def check_reindex():
|
||||
@shared_task
|
||||
def run_manual_import():
|
||||
"""called from settings page, to go through import folder"""
|
||||
|
||||
print("starting media file import")
|
||||
have_lock = False
|
||||
my_lock = RedisArchivist().get_lock("manual_import")
|
||||
@@ -111,7 +114,8 @@ def run_manual_import():
|
||||
if have_lock:
|
||||
import_handler = ManualImport()
|
||||
if import_handler.identified:
|
||||
import_handler.process_import()
|
||||
all_videos_added = import_handler.process_import()
|
||||
ThumbManager().download_vid(all_videos_added)
|
||||
else:
|
||||
print("Did not acquire lock form import.")
|
||||
|
||||
@@ -136,7 +140,9 @@ def run_restore_backup():
|
||||
|
||||
def kill_dl(task_id):
|
||||
"""kill download worker task by ID"""
|
||||
app.control.revoke(task_id, terminate=True)
|
||||
if task_id:
|
||||
app.control.revoke(task_id, terminate=True)
|
||||
|
||||
_ = RedisArchivist().del_message("dl_queue_id")
|
||||
RedisQueue("dl_queue").clear()
|
||||
|
||||
@@ -154,3 +160,9 @@ def kill_dl(task_id):
|
||||
"message": "",
|
||||
}
|
||||
RedisArchivist().set_message("progress:download", mess_dict)
|
||||
|
||||
|
||||
@shared_task
|
||||
def rescan_filesystem():
|
||||
"""check the media folder for mismatches"""
|
||||
scan_filesystem()
|
||||
|
||||
@@ -96,7 +96,7 @@
|
||||
</div>
|
||||
<div class="footer">
|
||||
<div class="boxed-content">
|
||||
<span>© 2021 The Tube Archivist v0.0.5 | <a href="https://github.com/bbilly1/tubearchivist" target="_blank">Github</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a></span>
|
||||
<span>© 2021 The Tube Archivist v0.0.6 | <a href="https://github.com/bbilly1/tubearchivist" target="_blank">Github</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a></span>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
|
||||
@@ -54,13 +54,11 @@
|
||||
{% if channels %}
|
||||
{% for channel in channels %}
|
||||
<div class="channel-item {{ view_style }}">
|
||||
{% if channel.source.channel_banner_url %}
|
||||
<div class="channel-banner {{ view_style }}">
|
||||
<a href="{% url 'channel_id' channel.source.channel_id %}">
|
||||
<img src="/cache/channels/{{ channel.source.channel_id }}_banner.jpg" alt="{{ channel.source.channel_id }}-banner">
|
||||
</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
<div class="channel-banner {{ view_style }}">
|
||||
<a href="{% url 'channel_id' channel.source.channel_id %}">
|
||||
<img src="/cache/channels/{{ channel.source.channel_id }}_banner.jpg" alt="{{ channel.source.channel_id }}-banner">
|
||||
</a>
|
||||
</div>
|
||||
<div class="info-box info-box-2 {{ view_style }}">
|
||||
<div class="info-box-item">
|
||||
<div class="round-img">
|
||||
|
||||
@@ -6,9 +6,7 @@
|
||||
<h1>Channel: {{ channel_info.channel_name }}</h1>
|
||||
</div>
|
||||
<div class="channel-banner">
|
||||
{% if channel_info.channel_banner_url %}
|
||||
<a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
|
||||
{% endif %}
|
||||
<a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
|
||||
</div>
|
||||
<div class="info-box info-box-3">
|
||||
<div class="info-box-item">
|
||||
@@ -38,20 +36,24 @@
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
{% if videos %}
|
||||
<div class="info-box-item">
|
||||
<div>
|
||||
{% if channel_info.channel_views >= 1000000 %}
|
||||
<p>Channel views: {{ channel_info.channel_views|intword }}</p>
|
||||
{% else %}
|
||||
{% elif channel_info.channel_views > 0 %}
|
||||
<p>Channel views: {{ channel_info.channel_views|intcomma }}</p>
|
||||
{% endif %}
|
||||
<p>Total Videos archived: {{ max_hits }}</p>
|
||||
<p>Watched: <button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="{{ channel_info.channel_id }}" onclick="isWatched(this.id)">Mark as watched</button></p>
|
||||
{% if max_hits %}
|
||||
<p>Total Videos archived: {{ max_hits }}</p>
|
||||
<p>Watched: <button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="{{ channel_info.channel_id }}" onclick="isWatched(this.id)">Mark as watched</button></p>
|
||||
{% endif %}
|
||||
<p>Channel id: {{ channel_info.channel_id }}</p>
|
||||
<button onclick="deleteConfirm()" id="delete-item">Delete Channel</button>
|
||||
<div class="delete-confirm" id="delete-button">
|
||||
<span>Delete {{ channel_info.channel_name }} including all videos? </span><button class="danger-button" onclick="deleteChannel(this)" data-id="{{ channel_info.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if channel_info.channel_description %}
|
||||
<div class="info-box-item description-box">
|
||||
@@ -71,10 +73,10 @@
|
||||
{% if videos %}
|
||||
{% for video in videos %}
|
||||
<div class="video-item {{ view_style }}">
|
||||
<a href="#player" data-src="/media/{{ video.source.media_url }}" data-thumb="/cache/videos/{{ video.source.youtube_id }}.jpg" data-title="{{ video.source.title }}" data-channel="{{ video.source.channel.channel_name }}" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
||||
<a href="#player" data-src="/media/{{ video.source.media_url }}" data-thumb="/cache/{{ video.source.vid_thumb_url }}" data-title="{{ video.source.title }}" data-channel="{{ video.source.channel.channel_name }}" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
||||
<div class="video-thumb-wrap {{ view_style }}">
|
||||
<div class="video-thumb">
|
||||
<img src="/cache/videos/{{ video.source.youtube_id }}.jpg" alt="video-thumb">
|
||||
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
</div>
|
||||
<div class="video-play">
|
||||
<img src="{% static 'img/icon-play.svg' %}" alt="play-icon">
|
||||
@@ -84,7 +86,7 @@
|
||||
<div class="video-desc {{ view_style }}">
|
||||
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
|
||||
{% if video.source.player.watched %}
|
||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" class="seen-icon">
|
||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
|
||||
{% else %}
|
||||
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.source.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
|
||||
{% endif %}
|
||||
|
||||
@@ -57,7 +57,7 @@
|
||||
{% for video in all_video_hits %}
|
||||
<div class="dl-item {{ view_style }}" id="dl-{{ video.youtube_id }}">
|
||||
<div class="dl-thumb {{ view_style }}">
|
||||
<img src="{{ video.vid_thumb_url }}" alt="video_thumb">
|
||||
<img src="/cache/{{ video.vid_thumb_url }}" alt="video_thumb">
|
||||
</div>
|
||||
<div class="dl-desc {{ view_style }}">
|
||||
{% if show_ignored_only %}
|
||||
|
||||
@@ -48,10 +48,10 @@
|
||||
{% if videos %}
|
||||
{% for video in videos %}
|
||||
<div class="video-item {{ view_style }}">
|
||||
<a href="#player" data-src="/media/{{ video.source.media_url }}" data-thumb="/cache/videos/{{ video.source.youtube_id }}.jpg" data-title="{{ video.source.title }}" data-channel="{{ video.source.channel.channel_name }}" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
||||
<a href="#player" data-src="/media/{{ video.source.media_url }}" data-thumb="/cache/{{ video.source.vid_thumb_url }}" data-title="{{ video.source.title }}" data-channel="{{ video.source.channel.channel_name }}" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
||||
<div class="video-thumb-wrap {{ view_style }}">
|
||||
<div class="video-thumb">
|
||||
<img src="/cache/videos/{{ video.source.youtube_id }}.jpg" alt="video-thumb">
|
||||
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
</div>
|
||||
<div class="video-play">
|
||||
<img src="{% static 'img/icon-play.svg' %}" alt="play-icon">
|
||||
@@ -61,7 +61,7 @@
|
||||
<div class="video-desc {{ view_style }}">
|
||||
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
|
||||
{% if video.source.player.watched %}
|
||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" class="seen-icon">
|
||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
|
||||
{% else %}
|
||||
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.source.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
|
||||
{% endif %}
|
||||
|
||||
@@ -88,6 +88,15 @@
|
||||
<option value="1">embed metadata</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="settings-item">
|
||||
<p>Current thumbnail embed setting: <span class="settings-current">{{ config.downloads.add_thumbnail }}</span></p>
|
||||
<i>Embed thumbnail into the mediafile.</i><br>
|
||||
<select name="downloads.add_thumbnail" id="downloads.add_thumbnail"">
|
||||
<option value="" disabled selected> -- change thumbnail embed -- </option>
|
||||
<option value="0">don't embed thumbnail</option>
|
||||
<option value="1">embed thumbnail</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<button type="submit">Update Settings</button>
|
||||
</form>
|
||||
@@ -96,7 +105,7 @@
|
||||
</div>
|
||||
<div class="settings-group">
|
||||
<h2>Manual media files import.</h2>
|
||||
<p>Add files to the <span class="settings-current">cache/import</span> folder. Make sure to follow the instructions on <a href="https://github.com/bbilly1/tubearchivist#import-your-existing-library" target="_blank">Github</a>.</p>
|
||||
<p>Add files to the <span class="settings-current">cache/import</span> folder. Make sure to follow the instructions in the Github <a href="https://github.com/bbilly1/tubearchivist/wiki/Settings" target="_blank">Wiki</a>.</p>
|
||||
<div id="manual-import">
|
||||
<button onclick="manualImport()">Start import</button>
|
||||
</div>
|
||||
@@ -117,7 +126,11 @@
|
||||
</div>
|
||||
</div>
|
||||
<div class="settings-group">
|
||||
<p>Rescan filesystem.</p>
|
||||
<i>Coming soon</i>
|
||||
<h2>Rescan filesystem</h2>
|
||||
<p><span class="danger-zone">Danger Zone</span>: This will delete the metadata of deleted videos from the filesystem.</p>
|
||||
<p>Rescan your media folder looking for missing videos and clean up index. More infos on the Github <a href="https://github.com/bbilly1/tubearchivist/wiki/Settings" target="_blank">Wiki</a>.</p>
|
||||
<div id="fs-rescan">
|
||||
<button onclick="fsRescan()">Rescan filesystem</button>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock content %}
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<div class="video-main">
|
||||
<video
|
||||
src="/media/{{ video.media_url }}"
|
||||
poster="/cache/videos/{{ video.youtube_id }}.jpg" controls preload="false"
|
||||
poster="/cache/{{ video.vid_thumb_url }}" controls preload="false"
|
||||
type='video/mp4' width="100%">
|
||||
</video>
|
||||
</div>
|
||||
@@ -34,7 +34,7 @@
|
||||
<p>Last refreshed: {{ video.vid_last_refresh }}</p>
|
||||
<p class="video-info-watched">Watched:
|
||||
{% if video.player.watched %}
|
||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" class="seen-icon" id="{{ video.youtube_id }}">
|
||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
|
||||
{% else %}
|
||||
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
|
||||
{% endif %}
|
||||
@@ -45,6 +45,10 @@
|
||||
<p>Youtube: Deactivated</p>
|
||||
{% endif %}
|
||||
<p>Video ID: {{ video.youtube_id }}</p>
|
||||
<button onclick="deleteConfirm()" id="delete-item">Delete Video</button>
|
||||
<div class="delete-confirm" id="delete-button">
|
||||
<span>Are you sure? </span><button class="danger-button" onclick="deleteVideo(this)" data-id="{{ video.youtube_id }}" data-redirect = "{{ video.channel.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="info-box-item">
|
||||
@@ -52,7 +56,13 @@
|
||||
<p>Views: {{ video.stats.view_count|intcomma }}</p>
|
||||
<p class="thumb-icon like"><img src="{% static 'img/icon-thumb.svg' %}" alt="thumbs-up">: {{ video.stats.like_count|intcomma }}</p>
|
||||
<p class="thumb-icon dislike"><img src="{% static 'img/icon-thumb.svg' %}" alt="thumbs-down">: {{ video.stats.dislike_count|intcomma }}</p>
|
||||
<p>Rating: {{ video.stats.average_rating }}</p>
|
||||
{% if video.stats.average_rating %}
|
||||
<p class="rating-stars">Rating:
|
||||
{% for star in video.stats.average_rating %}
|
||||
<img src="/static/img/icon-star-{{ star }}.svg" alt="{{ star }}">
|
||||
{% endfor %}
|
||||
</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
from django.test import TestCase # noqa: F401 - Unused import
|
||||
|
||||
# Create your tests here.
|
||||
0
tubearchivist/home/tests/__init__.py
Normal file
0
tubearchivist/home/tests/__init__.py
Normal file
42
tubearchivist/home/tests/test_views.py
Normal file
42
tubearchivist/home/tests/test_views.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""All test classes"""
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
|
||||
class URLTests(TestCase):
|
||||
"""test if all expected URL are there"""
|
||||
|
||||
def test_home_view(self):
|
||||
"""check homepage"""
|
||||
response = self.client.get("/")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
|
||||
def test_about_view(self):
|
||||
"""check about page"""
|
||||
response = self.client.get("/about/")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
|
||||
def test_downloads_view(self):
|
||||
"""check downloads page"""
|
||||
response = self.client.get("/downloads/")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
|
||||
def test_channel_view(self):
|
||||
"""check channel page"""
|
||||
response = self.client.get("/channel/")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
|
||||
def test_settings_view(self):
|
||||
"""check settings page"""
|
||||
response = self.client.get("/settings/")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
|
||||
def test_progress_view(self):
|
||||
"""check ajax progress endpoint"""
|
||||
response = self.client.get("/downloads/progress/")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
|
||||
def test_process_view(self):
|
||||
"""check process ajax endpoint"""
|
||||
response = self.client.get("/process/")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
@@ -19,7 +19,7 @@ urlpatterns = [
|
||||
path("downloads/", DownloadView.as_view(), name="downloads"),
|
||||
path("settings/", SettingsView.as_view(), name="settings"),
|
||||
path("process/", views.process, name="process"),
|
||||
path("downloads/progress", views.progress, name="progress"),
|
||||
path("downloads/progress/", views.progress, name="progress"),
|
||||
path("channel/", ChannelView.as_view(), name="channel"),
|
||||
path(
|
||||
"channel/<slug:channel_id_detail>/",
|
||||
|
||||
@@ -15,13 +15,14 @@ from django.views import View
|
||||
from home.src.config import AppConfig
|
||||
from home.src.download import ChannelSubscription, PendingList
|
||||
from home.src.helper import RedisArchivist, RedisQueue, process_url_list
|
||||
from home.src.index import WatchState
|
||||
from home.src.index import WatchState, YoutubeChannel, YoutubeVideo
|
||||
from home.src.searching import Pagination, SearchForm, SearchHandler
|
||||
from home.tasks import (
|
||||
download_pending,
|
||||
download_single,
|
||||
extrac_dl,
|
||||
kill_dl,
|
||||
rescan_filesystem,
|
||||
run_backup,
|
||||
run_manual_import,
|
||||
run_restore_backup,
|
||||
@@ -157,7 +158,7 @@ class DownloadView(View):
|
||||
|
||||
url = config["application"]["es_url"] + "/ta_download/_search"
|
||||
data = self.build_data(pagination_handler, show_ignored_only)
|
||||
search = SearchHandler(url, data, cache=False)
|
||||
search = SearchHandler(url, data)
|
||||
|
||||
videos_hits = search.get_data()
|
||||
max_hits = search.max_hits
|
||||
@@ -407,6 +408,12 @@ class VideoView(View):
|
||||
look_up = SearchHandler(url, data)
|
||||
video_hit = look_up.get_data()
|
||||
video_data = video_hit[0]["source"]
|
||||
try:
|
||||
rating = video_data["stats"]["average_rating"]
|
||||
video_data["stats"]["average_rating"] = self.star_creator(rating)
|
||||
except KeyError:
|
||||
video_data["stats"]["average_rating"] = False
|
||||
|
||||
video_title = video_data["title"]
|
||||
context = {"video": video_data, "title": video_title, "colors": colors}
|
||||
return render(request, "home/video.html", context)
|
||||
@@ -419,6 +426,20 @@ class VideoView(View):
|
||||
colors = config["application"]["colors"]
|
||||
return es_url, colors
|
||||
|
||||
@staticmethod
|
||||
def star_creator(rating):
|
||||
"""convert rating float to stars"""
|
||||
stars = []
|
||||
for _ in range(1, 6):
|
||||
if rating >= 0.75:
|
||||
stars.append("full")
|
||||
elif 0.25 < rating < 0.75:
|
||||
stars.append("half")
|
||||
else:
|
||||
stars.append("empty")
|
||||
rating = rating - 1
|
||||
return stars
|
||||
|
||||
|
||||
class SettingsView(View):
|
||||
"""resolves to /settings/
|
||||
@@ -489,6 +510,7 @@ class PostData:
|
||||
"""map dict key and return function to execute"""
|
||||
exec_map = {
|
||||
"watched": self.watched,
|
||||
"un_watched": self.un_watched,
|
||||
"change_view": self.change_view,
|
||||
"rescan_pending": self.rescan_pending,
|
||||
"ignore": self.ignore,
|
||||
@@ -505,7 +527,10 @@ class PostData:
|
||||
"manual-import": self.manual_import,
|
||||
"db-backup": self.db_backup,
|
||||
"db-restore": self.db_restore,
|
||||
"fs-rescan": self.fs_rescan,
|
||||
"channel-search": self.channel_search,
|
||||
"delete-video": self.delete_video,
|
||||
"delete-channel": self.delete_channel,
|
||||
}
|
||||
|
||||
return exec_map[self.to_exec]
|
||||
@@ -515,6 +540,11 @@ class PostData:
|
||||
WatchState(self.exec_val).mark_as_watched()
|
||||
return {"success": True}
|
||||
|
||||
def un_watched(self):
|
||||
"""mark as unwatched"""
|
||||
WatchState(self.exec_val).mark_as_unwatched()
|
||||
return {"success": True}
|
||||
|
||||
def change_view(self):
|
||||
"""process view changes in home, channel, and downloads"""
|
||||
origin, new_view = self.exec_val.split(":")
|
||||
@@ -558,7 +588,10 @@ class PostData:
|
||||
RedisQueue("dl_queue").clear()
|
||||
elif to_execute == "kill":
|
||||
task_id = RedisArchivist().get_message("dl_queue_id")
|
||||
print("brutally killing " + task_id)
|
||||
if not isinstance(task_id, str):
|
||||
task_id = False
|
||||
else:
|
||||
print("brutally killing " + task_id)
|
||||
kill_dl(task_id)
|
||||
|
||||
return {"success": True}
|
||||
@@ -652,9 +685,28 @@ class PostData:
|
||||
run_restore_backup.delay()
|
||||
return {"success": True}
|
||||
|
||||
@staticmethod
|
||||
def fs_rescan():
|
||||
"""start file system rescan task"""
|
||||
print("start filesystem scan")
|
||||
rescan_filesystem.delay()
|
||||
return {"success": True}
|
||||
|
||||
def channel_search(self):
|
||||
"""search for channel name as_you_type"""
|
||||
search_query = self.exec_val
|
||||
print("searching for: " + search_query)
|
||||
search_results = SearchForm().search_channels(search_query)
|
||||
return search_results
|
||||
|
||||
def delete_video(self):
|
||||
"""delete media file, metadata and thumb"""
|
||||
youtube_id = self.exec_val
|
||||
YoutubeVideo(youtube_id).delete_media_file()
|
||||
return {"success": True}
|
||||
|
||||
def delete_channel(self):
|
||||
"""delete channel and all matching videos"""
|
||||
channel_id = self.exec_val
|
||||
YoutubeChannel(channel_id).delete_channel()
|
||||
return {"success": True}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
beautifulsoup4==4.10.0
|
||||
celery==5.1.2
|
||||
Django==3.2.7
|
||||
Pillow==8.3.2
|
||||
Django==3.2.8
|
||||
Pillow==8.4.0
|
||||
redis==3.5.3
|
||||
requests==2.26.0
|
||||
uWSGI==2.0.19.1
|
||||
uWSGI==2.0.20
|
||||
whitenoise==5.3.0
|
||||
yt_dlp==2021.9.25
|
||||
yt_dlp==2021.10.10
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
--main-bg: #00202f;
|
||||
--highlight-bg: #00293b;
|
||||
--highlight-error: #990202;
|
||||
--highlight-error-light: #c44343;
|
||||
--highlight-bg-transparent: #00293baf;
|
||||
--main-font: #eeeeee;
|
||||
--accent-font-dark: #259485;
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
:root {
|
||||
--main-bg: #eeeeee;
|
||||
--highlight-bg: #d9e0d9;
|
||||
--highlight-error: #f79c9c;
|
||||
--highlight-error: #990202;
|
||||
--highlight-error-light: #c44343;
|
||||
--highlight-bg-transparent: #00293baf;
|
||||
--main-font: #00202f;
|
||||
--accent-font-dark: #259485;
|
||||
--accent-font-light: #35b399;
|
||||
--img-filter: invert(50%) sepia(9%) saturate(2940%) hue-rotate(122deg) brightness(94%) contrast(90%);
|
||||
--img-filter-error: invert(83%) sepia(35%) saturate(1238%) hue-rotate(297deg) brightness(103%) contrast(97%);
|
||||
--img-filter-error: invert(16%) sepia(60%) saturate(3717%) hue-rotate(349deg) brightness(86%) contrast(120%);
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
@font-face {
|
||||
font-family: 'Sen-Bold';
|
||||
src: url('../font/Sen-Bold.ttf.woff');
|
||||
src: url('../font/Sen-Bold.woff');
|
||||
font-family: 'Sen-Bold';
|
||||
}
|
||||
|
||||
@font-face {
|
||||
font-family: 'Sen-Regular';
|
||||
src: url('../font/Sen-Regular.ttf.woff');
|
||||
src: url('../font/Sen-Regular.woff');
|
||||
font-family: 'Sen-Regular';
|
||||
}
|
||||
|
||||
@@ -205,6 +205,19 @@ button:hover {
|
||||
color: var(--main-font);
|
||||
}
|
||||
|
||||
/* delete button */
|
||||
.delete-confirm {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.danger-button {
|
||||
background-color: var(--highlight-error);
|
||||
}
|
||||
|
||||
.danger-button:hover {
|
||||
background-color: var(--highlight-error-light);
|
||||
}
|
||||
|
||||
/* navigation */
|
||||
.top-nav {
|
||||
display: block;
|
||||
@@ -440,15 +453,12 @@ button:hover {
|
||||
}
|
||||
|
||||
.unseen-icon,
|
||||
.seen-icon,
|
||||
.close-button {
|
||||
cursor: pointer;
|
||||
filter: var(--img-filter);
|
||||
}
|
||||
|
||||
.seen-icon {
|
||||
filter: var(--img-filter);
|
||||
}
|
||||
|
||||
.video-more {
|
||||
text-decoration: underline;
|
||||
text-align: right;
|
||||
@@ -518,7 +528,8 @@ button:hover {
|
||||
margin-left: 5px;
|
||||
}
|
||||
|
||||
.thumb-icon img {
|
||||
.thumb-icon img,
|
||||
.rating-stars img {
|
||||
width: 20px;
|
||||
margin: 0;
|
||||
filter: var(--img-filter);
|
||||
@@ -661,6 +672,10 @@ button:hover {
|
||||
background-color: var(--highlight-error);
|
||||
}
|
||||
|
||||
.download-progress.error h3 {
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
/* settings */
|
||||
.settings-group {
|
||||
background-color: var(--highlight-bg);
|
||||
@@ -678,6 +693,7 @@ button:hover {
|
||||
|
||||
.danger-zone {
|
||||
background-color: var(--highlight-error);
|
||||
color: #fff;
|
||||
padding: 3px;
|
||||
}
|
||||
|
||||
|
||||
94
tubearchivist/static/font/OFL_License.txt
Normal file
94
tubearchivist/static/font/OFL_License.txt
Normal file
@@ -0,0 +1,94 @@
|
||||
Copyright (c) 2015, Kosal Sen, Philatype (<http://philatype.com>),
|
||||
with Reserved Font Name Sen.
|
||||
|
||||
This Font Software is licensed under the SIL Open Font License, Version 1.1.
|
||||
This license is copied below, and is also available with a FAQ at:
|
||||
http://scripts.sil.org/OFL
|
||||
|
||||
|
||||
-----------------------------------------------------------
|
||||
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
|
||||
-----------------------------------------------------------
|
||||
|
||||
PREAMBLE
|
||||
The goals of the Open Font License (OFL) are to stimulate worldwide
|
||||
development of collaborative font projects, to support the font creation
|
||||
efforts of academic and linguistic communities, and to provide a free and
|
||||
open framework in which fonts may be shared and improved in partnership
|
||||
with others.
|
||||
|
||||
The OFL allows the licensed fonts to be used, studied, modified and
|
||||
redistributed freely as long as they are not sold by themselves. The
|
||||
fonts, including any derivative works, can be bundled, embedded,
|
||||
redistributed and/or sold with any software provided that any reserved
|
||||
names are not used by derivative works. The fonts and derivatives,
|
||||
however, cannot be released under any other type of license. The
|
||||
requirement for fonts to remain under this license does not apply
|
||||
to any document created using the fonts or their derivatives.
|
||||
|
||||
DEFINITIONS
|
||||
"Font Software" refers to the set of files released by the Copyright
|
||||
Holder(s) under this license and clearly marked as such. This may
|
||||
include source files, build scripts and documentation.
|
||||
|
||||
"Reserved Font Name" refers to any names specified as such after the
|
||||
copyright statement(s).
|
||||
|
||||
"Original Version" refers to the collection of Font Software components as
|
||||
distributed by the Copyright Holder(s).
|
||||
|
||||
"Modified Version" refers to any derivative made by adding to, deleting,
|
||||
or substituting -- in part or in whole -- any of the components of the
|
||||
Original Version, by changing formats or by porting the Font Software to a
|
||||
new environment.
|
||||
|
||||
"Author" refers to any designer, engineer, programmer, technical
|
||||
writer or other person who contributed to the Font Software.
|
||||
|
||||
PERMISSION & CONDITIONS
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of the Font Software, to use, study, copy, merge, embed, modify,
|
||||
redistribute, and sell modified and unmodified copies of the Font
|
||||
Software, subject to the following conditions:
|
||||
|
||||
1) Neither the Font Software nor any of its individual components,
|
||||
in Original or Modified Versions, may be sold by itself.
|
||||
|
||||
2) Original or Modified Versions of the Font Software may be bundled,
|
||||
redistributed and/or sold with any software, provided that each copy
|
||||
contains the above copyright notice and this license. These can be
|
||||
included either as stand-alone text files, human-readable headers or
|
||||
in the appropriate machine-readable metadata fields within text or
|
||||
binary files as long as those fields can be easily viewed by the user.
|
||||
|
||||
3) No Modified Version of the Font Software may use the Reserved Font
|
||||
Name(s) unless explicit written permission is granted by the corresponding
|
||||
Copyright Holder. This restriction only applies to the primary font name as
|
||||
presented to the users.
|
||||
|
||||
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
||||
Software shall not be used to promote, endorse or advertise any
|
||||
Modified Version, except to acknowledge the contribution(s) of the
|
||||
Copyright Holder(s) and the Author(s) or with their explicit written
|
||||
permission.
|
||||
|
||||
5) The Font Software, modified or unmodified, in part or in whole,
|
||||
must be distributed entirely under this license, and must not be
|
||||
distributed under any other license. The requirement for fonts to
|
||||
remain under this license does not apply to any document created
|
||||
using the Font Software.
|
||||
|
||||
TERMINATION
|
||||
This license becomes null and void if any of the above conditions are
|
||||
not met.
|
||||
|
||||
DISCLAIMER
|
||||
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
||||
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
||||
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
||||
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
|
||||
OTHER DEALINGS IN THE FONT SOFTWARE.
|
||||
BIN
tubearchivist/static/font/Sen-Bold.woff
Normal file
BIN
tubearchivist/static/font/Sen-Bold.woff
Normal file
Binary file not shown.
BIN
tubearchivist/static/font/Sen-Regular.woff
Normal file
BIN
tubearchivist/static/font/Sen-Regular.woff
Normal file
Binary file not shown.
BIN
tubearchivist/static/img/default-channel-banner.jpg
Normal file
BIN
tubearchivist/static/img/default-channel-banner.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
BIN
tubearchivist/static/img/default-channel-icon.jpg
Normal file
BIN
tubearchivist/static/img/default-channel-icon.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 84 KiB |
BIN
tubearchivist/static/img/default-video-thumb.jpg
Normal file
BIN
tubearchivist/static/img/default-video-thumb.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 56 KiB |
65
tubearchivist/static/img/icon-star-empty.svg
Normal file
65
tubearchivist/static/img/icon-star-empty.svg
Normal file
@@ -0,0 +1,65 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!-- Created with Inkscape (http://www.inkscape.org/) -->
|
||||
|
||||
<svg
|
||||
width="500"
|
||||
height="500"
|
||||
viewBox="0 0 132.29197 132.29167"
|
||||
version="1.1"
|
||||
id="svg1303"
|
||||
inkscape:version="1.1.1 (3bf5ae0d25, 2021-09-20)"
|
||||
sodipodi:docname="icon-star-empty.svg"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:svg="http://www.w3.org/2000/svg"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<defs
|
||||
id="defs1297" />
|
||||
<sodipodi:namedview
|
||||
id="base"
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#666666"
|
||||
borderopacity="1.0"
|
||||
inkscape:pageopacity="0.0"
|
||||
inkscape:pageshadow="2"
|
||||
inkscape:zoom="2.4005076"
|
||||
inkscape:cx="40.40812"
|
||||
inkscape:cy="211.20533"
|
||||
inkscape:document-units="mm"
|
||||
inkscape:current-layer="layer1"
|
||||
showgrid="false"
|
||||
units="px"
|
||||
inkscape:window-width="3840"
|
||||
inkscape:window-height="2112"
|
||||
inkscape:window-x="0"
|
||||
inkscape:window-y="48"
|
||||
inkscape:window-maximized="1"
|
||||
inkscape:pagecheckerboard="0" />
|
||||
<metadata
|
||||
id="metadata1300">
|
||||
<rdf:RDF>
|
||||
<cc:Work
|
||||
rdf:about="">
|
||||
<dc:format>image/svg+xml</dc:format>
|
||||
<dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
||||
<dc:title />
|
||||
</cc:Work>
|
||||
</rdf:RDF>
|
||||
</metadata>
|
||||
<g
|
||||
inkscape:label="Ebene 1"
|
||||
inkscape:groupmode="layer"
|
||||
id="layer1"
|
||||
transform="translate(0,-164.70764)">
|
||||
<path
|
||||
id="path1255"
|
||||
style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
|
||||
inkscape:transform-center-y="-6.4703827"
|
||||
d="M 250 21.158203 L 189.78906 194.37891 L 6.4375 198.11523 L 152.57617 308.9082 L 99.470703 484.43945 L 250 379.69141 L 400.5293 484.43945 L 347.42383 308.9082 L 493.5625 198.11523 L 310.21094 194.37891 L 250 21.158203 z M 250 96.408203 L 291.69141 216.34766 L 418.64453 218.93555 L 317.45703 295.65039 L 354.22852 417.18945 L 250 344.66211 L 145.77148 417.18945 L 182.54297 295.65039 L 81.355469 218.93555 L 208.30859 216.34766 L 250 96.408203 z "
|
||||
transform="matrix(0.26458394,0,0,0.26458394,0,164.70749)" />
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.5 KiB |
74
tubearchivist/static/img/icon-star-full.svg
Normal file
74
tubearchivist/static/img/icon-star-full.svg
Normal file
@@ -0,0 +1,74 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!-- Created with Inkscape (http://www.inkscape.org/) -->
|
||||
|
||||
<svg
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:svg="http://www.w3.org/2000/svg"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
width="500"
|
||||
height="500"
|
||||
viewBox="0 0 132.29197 132.29167"
|
||||
version="1.1"
|
||||
id="svg1303"
|
||||
inkscape:version="0.92.4 (5da689c313, 2019-01-14)"
|
||||
sodipodi:docname="Icons_star.svg">
|
||||
<defs
|
||||
id="defs1297" />
|
||||
<sodipodi:namedview
|
||||
id="base"
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#666666"
|
||||
borderopacity="1.0"
|
||||
inkscape:pageopacity="0.0"
|
||||
inkscape:pageshadow="2"
|
||||
inkscape:zoom="1.0105705"
|
||||
inkscape:cx="61.891881"
|
||||
inkscape:cy="148.25167"
|
||||
inkscape:document-units="mm"
|
||||
inkscape:current-layer="layer1"
|
||||
showgrid="false"
|
||||
units="px"
|
||||
inkscape:window-width="1920"
|
||||
inkscape:window-height="1017"
|
||||
inkscape:window-x="-8"
|
||||
inkscape:window-y="-8"
|
||||
inkscape:window-maximized="1" />
|
||||
<metadata
|
||||
id="metadata1300">
|
||||
<rdf:RDF>
|
||||
<cc:Work
|
||||
rdf:about="">
|
||||
<dc:format>image/svg+xml</dc:format>
|
||||
<dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
||||
<dc:title />
|
||||
</cc:Work>
|
||||
</rdf:RDF>
|
||||
</metadata>
|
||||
<g
|
||||
inkscape:label="Ebene 1"
|
||||
inkscape:groupmode="layer"
|
||||
id="layer1"
|
||||
transform="translate(0,-164.70764)">
|
||||
<path
|
||||
sodipodi:type="star"
|
||||
style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
|
||||
id="path1255"
|
||||
sodipodi:sides="5"
|
||||
sodipodi:cx="66.145981"
|
||||
sodipodi:cy="237.32387"
|
||||
sodipodi:r1="67.758904"
|
||||
sodipodi:r2="27.103561"
|
||||
sodipodi:arg1="-1.5707963"
|
||||
sodipodi:arg2="-0.9424778"
|
||||
inkscape:flatsided="false"
|
||||
inkscape:rounded="-3.46945e-18"
|
||||
inkscape:randomized="0"
|
||||
d="m 66.145983,169.56496 15.931071,45.83167 48.511476,0.98859 L 91.923,245.69933 105.97366,292.14197 66.145981,264.42743 26.318295,292.14197 40.368962,245.69933 1.7034347,216.38521 50.214907,215.39663 Z"
|
||||
inkscape:transform-center-y="-6.4703827" />
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.5 KiB |
66
tubearchivist/static/img/icon-star-half.svg
Normal file
66
tubearchivist/static/img/icon-star-half.svg
Normal file
@@ -0,0 +1,66 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!-- Created with Inkscape (http://www.inkscape.org/) -->
|
||||
|
||||
<svg
|
||||
width="500"
|
||||
height="500"
|
||||
viewBox="0 0 132.29197 132.29167"
|
||||
version="1.1"
|
||||
id="svg1303"
|
||||
inkscape:version="1.1.1 (3bf5ae0d25, 2021-09-20)"
|
||||
sodipodi:docname="icon-star-half.svg"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:svg="http://www.w3.org/2000/svg"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<defs
|
||||
id="defs1297" />
|
||||
<sodipodi:namedview
|
||||
id="base"
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#666666"
|
||||
borderopacity="1.0"
|
||||
inkscape:pageopacity="0.0"
|
||||
inkscape:pageshadow="2"
|
||||
inkscape:zoom="1.0105705"
|
||||
inkscape:cx="44.034533"
|
||||
inkscape:cy="168.22181"
|
||||
inkscape:document-units="mm"
|
||||
inkscape:current-layer="layer1"
|
||||
showgrid="false"
|
||||
units="px"
|
||||
inkscape:window-width="3840"
|
||||
inkscape:window-height="2078"
|
||||
inkscape:window-x="0"
|
||||
inkscape:window-y="82"
|
||||
inkscape:window-maximized="1"
|
||||
inkscape:pagecheckerboard="0" />
|
||||
<metadata
|
||||
id="metadata1300">
|
||||
<rdf:RDF>
|
||||
<cc:Work
|
||||
rdf:about="">
|
||||
<dc:format>image/svg+xml</dc:format>
|
||||
<dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
||||
<dc:title />
|
||||
</cc:Work>
|
||||
</rdf:RDF>
|
||||
</metadata>
|
||||
<g
|
||||
inkscape:label="Ebene 1"
|
||||
inkscape:groupmode="layer"
|
||||
id="layer1"
|
||||
transform="translate(0,-164.70764)">
|
||||
<path
|
||||
id="path1255"
|
||||
style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
|
||||
inkscape:transform-center-x="1.1220981e-06"
|
||||
inkscape:transform-center-y="-6.4703827"
|
||||
d="M 250 21.158203 L 189.78906 194.37891 L 6.4375 198.11523 L 152.57617 308.9082 L 99.470703 484.43945 L 250 379.69141 L 400.5293 484.43945 L 347.42383 308.9082 L 493.5625 198.11523 L 310.21094 194.37891 L 250 21.158203 z M 250 96.408203 L 291.69141 216.34766 L 418.64453 218.93555 L 317.45703 295.65039 L 354.22852 417.18945 L 250 344.66211 L 250 96.408203 z "
|
||||
transform="matrix(0.26458394,0,0,0.26458394,0,164.70749)" />
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.4 KiB |
@@ -32,7 +32,7 @@ function handleInterval() {
|
||||
var watchDownload = setInterval(function() {
|
||||
var req = new XMLHttpRequest();
|
||||
req.responseType = 'json';
|
||||
req.open('GET', '/downloads/progress', true);
|
||||
req.open('GET', '/downloads/progress/', true);
|
||||
req.onload = function() {
|
||||
var dlProgress = req.response;
|
||||
if (dlProgress['status']) {
|
||||
|
||||
@@ -15,10 +15,25 @@ function isWatched(youtube_id) {
|
||||
seenIcon.setAttribute('src', "/static/img/icon-seen.svg");
|
||||
seenIcon.setAttribute('alt', 'seen-icon');
|
||||
seenIcon.setAttribute('id', youtube_id);
|
||||
seenIcon.setAttribute('title', "Mark as unwatched");
|
||||
seenIcon.setAttribute('onclick', "isUnwatched(this.id)");
|
||||
seenIcon.classList = 'seen-icon';
|
||||
document.getElementById(youtube_id).replaceWith(seenIcon);
|
||||
}
|
||||
|
||||
function isUnwatched(youtube_id) {
|
||||
var payload = JSON.stringify({'un_watched': youtube_id});
|
||||
sendPost(payload);
|
||||
var unseenIcon = document.createElement('img');
|
||||
unseenIcon.setAttribute('src', "/static/img/icon-unseen.svg");
|
||||
unseenIcon.setAttribute('alt', 'unseen-icon');
|
||||
unseenIcon.setAttribute('id', youtube_id);
|
||||
unseenIcon.setAttribute('title', "Mark as watched");
|
||||
unseenIcon.setAttribute('onclick', "isWatched(this.id)");
|
||||
unseenIcon.classList = 'unseen-icon';
|
||||
document.getElementById(youtube_id).replaceWith(unseenIcon);
|
||||
}
|
||||
|
||||
function unsubscribe(channel_id) {
|
||||
var payload = JSON.stringify({'unsubscribe': channel_id});
|
||||
sendPost(payload);
|
||||
@@ -141,7 +156,7 @@ function dbBackup() {
|
||||
|
||||
function dbRestore() {
|
||||
var payload = JSON.stringify({'db-restore': true});
|
||||
sendPost(payload)
|
||||
sendPost(payload);
|
||||
// clear button
|
||||
var message = document.createElement('p');
|
||||
message.innerText = 'restoring from backup';
|
||||
@@ -150,6 +165,51 @@ function dbRestore() {
|
||||
toReplace.appendChild(message);
|
||||
}
|
||||
|
||||
function fsRescan() {
|
||||
var payload = JSON.stringify({'fs-rescan': true});
|
||||
sendPost(payload);
|
||||
// clear button
|
||||
var message = document.createElement('p');
|
||||
message.innerText = 'File system scan in progress';
|
||||
var toReplace = document.getElementById('fs-rescan');
|
||||
toReplace.innerHTML = '';
|
||||
toReplace.appendChild(message);
|
||||
}
|
||||
|
||||
// delete from file system
|
||||
function deleteConfirm() {
|
||||
to_show = document.getElementById("delete-button");
|
||||
document.getElementById("delete-item").style.display = 'none';
|
||||
to_show.style.display = "block";
|
||||
}
|
||||
|
||||
function deleteVideo(button) {
|
||||
var to_delete = button.getAttribute("data-id");
|
||||
var to_redirect = button.getAttribute("data-redirect");
|
||||
var payload = JSON.stringify({"delete-video": to_delete});
|
||||
sendPost(payload);
|
||||
setTimeout(function(){
|
||||
var redirect = "/channel/" + to_redirect;
|
||||
window.location.replace(redirect);
|
||||
return false;
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
function deleteChannel(button) {
|
||||
var to_delete = button.getAttribute("data-id");
|
||||
var payload = JSON.stringify({"delete-channel": to_delete});
|
||||
sendPost(payload);
|
||||
setTimeout(function(){
|
||||
window.location.replace("/channel/");
|
||||
return false;
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
function cancelDelete() {
|
||||
document.getElementById("delete-button").style.display = 'none';
|
||||
document.getElementById("delete-item").style.display = 'block';
|
||||
}
|
||||
|
||||
// player
|
||||
function createPlayer(button) {
|
||||
var mediaUrl = button.getAttribute('data-src');
|
||||
|
||||
Reference in New Issue
Block a user