Compare commits

...

48 Commits

Author SHA1 Message Date
simon
528938942a update docs for v0.0.6 take 2 2021-10-17 11:31:21 +07:00
simon
de2371c43c handle running empty dl queue, #55 2021-10-17 11:28:05 +07:00
simon
1a88c85436 skip video if is_live now 2021-10-17 10:49:02 +07:00
simon
63be6afcfe allow duration_sec extraction to fail cleanly for add to queue 2021-10-17 09:40:46 +07:00
simon
559a4c9a58 a note about multi arch builds 2021-10-16 18:12:41 +07:00
simon
08338f3270 add thumbnail download for manual import 2021-10-16 17:03:39 +07:00
simon
0c3c1b650a disable notifiation on rescan, fix keyerror issue in missing_thumbs build 2021-10-15 13:16:35 +07:00
simon
45d8b7e218 ignore missing video rating 2021-10-14 23:12:30 +07:00
simon
5edc78a826 update docs for v0.0.6 2021-10-14 22:48:43 +07:00
simon
fd2fa628e3 make sure to get both banner and icon 2021-10-14 22:48:23 +07:00
simon
ba12c2ad25 simple progress message for thumbnail bulk download 2021-10-14 21:28:27 +07:00
simon
0d061d8601 get art work for new subscribed channels 2021-10-14 17:47:07 +07:00
simon
0844f7b91a add delayed retry for non 404 thumb download error 2021-10-14 14:44:50 +07:00
simon
044b12d5a5 remove now unused thumbnail methods from searching module 2021-10-14 14:42:35 +07:00
simon
ce5f8b6a38 cleanup method for unused thumbs 2021-10-14 11:59:30 +07:00
simon
9ad42dfa38 add artwork refresh on rescan 2021-10-14 11:18:32 +07:00
simon
952bc4f97c add thumb dl after rescan 2021-10-14 10:03:54 +07:00
simon
bfd51e6804 extend test deploy to take platform argument 2021-10-13 19:34:55 +07:00
simon
72af95acd8 fallback channel and video artwork for 404 errors 2021-10-13 19:34:16 +07:00
simon
3473c79d63 use TARGETPLATFORM to install repo ffmpeg for arm64 and patched ffmpeg for amd64 builds 2021-10-12 23:00:25 +07:00
simon
a4ae6ca1ca Merge branch 'testing' of github.com:bbilly1/tubearchivist into testing 2021-10-12 17:49:18 +07:00
lamusmaser
48a4827ad5 Update deploy.sh to utilize buildx for multiarch builds. This is inline with Issue #6. (#51) 2021-10-12 17:47:48 +07:00
simon
29f17680bf consolidate delete thumbnails into ThumbManager class 2021-10-11 19:29:27 +07:00
simon
519c710e7d testing tests 2021-10-11 16:55:47 +07:00
simon
1b04c00ae7 standardizing on progress/ url 2021-10-11 16:55:08 +07:00
simon
9f12966ef7 bumping yt_dlp version, requires additional noprogress option 2021-10-11 16:34:19 +07:00
simon
76f49b4cf7 comparing with instance instead 2021-10-11 16:11:30 +07:00
simon
35a5eda36e downloading channel artwork after scraping 2021-10-11 16:03:25 +07:00
simon
567e9d473b fix killing task with missing task id 2021-10-11 15:54:36 +07:00
simon
e202265a5f extended validate_thumbnails for channel art 2021-10-11 14:27:48 +07:00
simon
bc84696792 new thumbnails module and new cache layout 2021-10-11 13:33:28 +07:00
simon
f5621954fb star icons for video rating 2021-10-10 16:09:02 +07:00
simon
ef75f6dd16 add delete channel to frontend 2021-10-09 20:33:32 +07:00
simon
b53a1763c8 better channel folder path builder with clean_string 2021-10-09 20:32:42 +07:00
simon
11780a0ff0 add delete channel backend 2021-10-09 19:54:36 +07:00
simon
8d3e2f14fc add delete video to frontend 2021-10-09 17:11:13 +07:00
simon
7405e960e5 delete video backend functionality 2021-10-08 16:18:01 +07:00
simon
8c97e6786a chown to fix permissions on new channel folders 2021-10-08 15:16:02 +07:00
simon
563222a26e fix linting 2021-10-08 15:10:44 +07:00
simon
64ccd3830e implementing filesystem rescan to clean index 2021-10-08 14:56:07 +07:00
simon
c2a6ac6f44 fix wiki link 2021-10-08 11:36:18 +07:00
simon
0b88fd8b1f implement watched-unwatched toggle, #39 2021-10-07 23:38:17 +07:00
simon
744780f4bd add optional EmbedThumbnail postprocessor 2021-10-07 22:28:32 +07:00
simon
4d152e1e54 bumping versions 2021-10-07 20:08:38 +07:00
simon
3500044b0d adding woff files and OFL license 2021-10-07 19:41:52 +07:00
simon
7e0abc0d20 fix run sync_redis_state on startup 2021-10-04 01:07:15 +07:00
simon
b0fb2bbb00 update roadmap 2021-10-03 20:25:35 +07:00
simon
4e7cba70d5 update to new ffmpeg naming convention 2021-10-03 20:25:23 +07:00
45 changed files with 1082 additions and 210 deletions

3
.gitignore vendored
View File

@@ -3,6 +3,3 @@ __pycache__
# django testing db
db.sqlite3
# frontend fonts
*ttf.woff

View File

@@ -27,6 +27,7 @@ This is my setup I have landed on, YMMV:
./deploy.sh test
```
- The command above will also copy the file `tubarchivist/testing.sh` into the working folder of the container. Running this script will install additional debugging tools I regularly use in testing.
- The `test` argument takes another optional argument to build for a specific architecture valid options are: `amd64`, `arm64` and `multi`, default is `amd64`.
- This `deploy.sh` file is not meant to be universally usable for every possible environment but could serve as an idea on how to automatically rebuild containers to test changes - customize to your liking.
## Implementing a new feature

View File

@@ -1,6 +1,7 @@
# build the tube archivist image from default python slim image
FROM python:3.9.7-slim-bullseye
ARG TARGETPLATFORM
ENV PYTHONUNBUFFERED 1
@@ -10,16 +11,19 @@ RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recomm
nginx \
curl && rm -rf /var/lib/apt/lists/*
# get newest patched ffmpeg and ffprobe builds
RUN curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
| grep browser_download_url \
| grep linux64 \
| grep ffmpeg-n \
| cut -d '"' -f 4 \
| xargs curl -L --output ffmpeg.tar.xz && \
tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \
rm ffmpeg.tar.xz
# get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64
RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
| grep browser_download_url \
| grep linux64-gpl-4.4.tar.xz \
| cut -d '"' -f 4 \
| xargs curl -L --output ffmpeg.tar.xz && \
tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \
rm ffmpeg.tar.xz \
; elif [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \
apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \
; fi
# copy config files
COPY nginx.conf /etc/nginx/conf.d/

View File

@@ -73,9 +73,15 @@ For some architectures it might be required to run Redis JSON on a nonstandard p
### Updating Tube Archivist
You will see the current version number of **Tube Archivist** in the footer of the interface so you can compare it with the latest release to make sure you are running the *latest and greatest*.
* There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. Any breaking changes will be marked in the **release notes**.
* There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.
* All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated when a new release of Elasticsearch is available. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned.
### Alternative installation instructions:
- **arm64**: Newest Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, you can use `bbilly1/rejson`, an unofficial rebuild for arm64.
- NOTE: This is untested, looking for feedback.
- **Synology**: There is a [discussion thread](https://github.com/bbilly1/tubearchivist/discussions/48) with Synology installation instructions.
- **Unraid**: There is a [template](https://github.com/pairofcrocs/unraid-templates/) and forum [thread](https://forums.unraid.net/topic/114073-support-crocs-tube-archivist/) with additional information.
## Potential pitfalls
### vm.max_map_count
**Elastic Search** in Docker requires the kernel setting of the host machine `vm.max_map_count` to be set to at least 262144.
@@ -111,22 +117,22 @@ This should be considered as a **minimal viable product**, there is an extensive
### Functionality
- [ ] Access control
- [ ] User roles
- [ ] Delete videos and channel
- [ ] Create playlists
- [ ] Podcast mode to serve channel as mp3
- [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage
- [ ] Un-ignore videos
- [ ] Add thumbnail embed option
- [X] Delete videos and channel [2021-10-16]
- [X] Add thumbnail embed option [2021-10-16]
- [X] Un-ignore videos [2021-10-03]
- [X] Dynamic download queue [2021-09-26]
- [X] Backup and restore [2021-09-22]
- [X] Scan your file system to index already downloaded videos [2021-09-14]
### UI
- [ ] Create a github wiki for user documentation
- [ ] Show similar videos on video page
- [ ] Multi language support
- [ ] Grid and list view for both channel and video list pages
- [ ] Show total video downloaded vs total videos available in channel
- [X] Grid and list view for both channel and video list pages [2021-10-03]
- [X] Create a github wiki for user documentation [2021-10-03]
## Known limitations

View File

@@ -5,11 +5,16 @@
# blackhole for local production
# docker to publish
# create builder:
# docker buildx create --name tubearchivist
# docker buildx use tubearchivist
# docker buildx inspect --bootstrap
set -e
function sync_blackhole {
# docker commands need sudo
# docker commands need sudo, only build amd64
host="blackhole.local"
read -sp 'Password: ' remote_pw
@@ -23,7 +28,7 @@ function sync_blackhole {
--exclude "db.sqlite3" \
. -e ssh "$host":tubearchivist
echo "$PASS" | ssh "$host" 'sudo -S docker build -t bbilly1/tubearchivist:latest tubearchivist 2>/dev/null'
echo "$PASS" | ssh "$host" 'sudo -S docker buildx build --platform linux/amd64 -t bbilly1/tubearchivist:latest tubearchivist --load 2>/dev/null'
echo "$PASS" | ssh "$host" 'sudo -S docker-compose up -d 2>/dev/null'
}
@@ -31,6 +36,8 @@ function sync_blackhole {
function sync_test {
# docker commands don't need sudo in testing vm
# pass argument to build for specific platform
host="tubearchivist.local"
rsync -a --progress --delete-after \
@@ -43,7 +50,17 @@ function sync_test {
rsync -r --progress --delete docker-compose.yml -e ssh "$host":docker
ssh "$host" 'docker build -t bbilly1/tubearchivist:latest tubearchivist'
if [[ $1 = "amd64" ]]; then
platform="linux/amd64"
elif [[ $1 = "arm64" ]]; then
platform="linux/arm64"
elif [[ $1 = "multi" ]]; then
platform="linux/amd64,linux/arm64"
else
platform="linux/amd64"
fi
ssh "$host" "docker buildx build --platform $platform -t bbilly1/tubearchivist:latest tubearchivist --load"
ssh "$host" 'docker-compose -f docker/docker-compose.yml up -d'
ssh "$host" 'docker cp tubearchivist/tubearchivist/testing.sh tubearchivist:/app/testing.sh'
@@ -103,7 +120,7 @@ function sync_docker {
read -r VERSION
# start build
sudo docker build -t bbilly1/tubearchivist:latest -t bbilly1/tubearchivist:"$VERSION" .
sudo docker buildx build --platform linux/amd64,linux/arm64 -t bbilly1/tubearchivist:latest -t bbilly1/tubearchivist:"$VERSION" .
printf "\nlatest images:\n"
sudo docker image ls bbilly1/tubearchivist
@@ -112,10 +129,10 @@ function sync_docker {
read -rn 1
# push to docker
echo "pushing latest:"
sudo docker push bbilly1/tubearchivist:latest
echo "pushing $VERSION"
sudo docker push bbilly1/tubearchivist:"$VERSION"
echo "pushing multiarch latest and $VERSION:"
sudo docker buildx build --platform linux/amd64,linux/arm64 -t bbilly1/tubearchivist:latest -t bbilly1/tubearchivist:"$VERSION" --push .
#sudo docker push bbilly1/tubearchivist:latest
#sudo docker push bbilly1/tubearchivist:"$VERSION"
# create release tag
echo "commits since last version:"
@@ -133,7 +150,7 @@ python version_check.py
if [[ $1 == "blackhole" ]]; then
sync_blackhole
elif [[ $1 == "test" ]]; then
sync_test
sync_test "$2"
elif [[ $1 == "validate" ]]; then
validate "$2"
elif [[ $1 == "docker" ]]; then

View File

@@ -29,7 +29,7 @@ services:
depends_on:
- archivist-es
archivist-es:
image: docker.elastic.co/elasticsearch/elasticsearch:7.15.0
image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
container_name: archivist-es
restart: always
environment:

View File

@@ -21,3 +21,4 @@ Each channel will get a dedicated channel detail page accessible at `/channel/<c
- If you are subscribed to the channel, an *Unsubscribe* button will show.
- You can *Show* the channel description, that matches with the *About* tab on YouTube.
- The **Mark as Watched** button will mark all videos of this channel as watched.
- The button **Delete Channel** will delete the channel plus all videos of this channel, both media files and metadata.

View File

@@ -1,7 +1,4 @@
# Tube Archivist Wiki
**WIP**: This is work in progress!
Welcome to the official Tube Archivist Wiki. This is an up-to-date documentation of user functionality.
Table of contents:
@@ -24,6 +21,6 @@ Table of contents:
* Hover over the playing video to show additional control options.
An empty checkbox icon <img src="assets/icon-unseen.png?raw=true" alt="unseen icon" width="20px" style="margin:0 5px;"> will show for videos you haven't marked as watched. Click on it and the icon will change to a filled checkbox <img src="assets/icon-seen.png?raw=true" alt="seen icon" width="20px" style="margin:0 5px;"> indicating it as watched.
An empty checkbox icon <img src="assets/icon-unseen.png?raw=true" alt="unseen icon" width="20px" style="margin:0 5px;"> will show for videos you haven't marked as watched. Click on it and the icon will change to a filled checkbox <img src="assets/icon-seen.png?raw=true" alt="seen icon" width="20px" style="margin:0 5px;"> indicating it as watched - click again to revert.
When available the <img src="assets/icon-gridview.png?raw=true" alt="gridview icon" width="20px" style="margin:0 5px;"> gridview icon will display the list in a grid, the <img src="assets/icon-listview.png?raw=true" alt="listview icon" width="20px" style="margin:0 5px;"> listview icon will arrange the items in a list.

View File

@@ -23,6 +23,7 @@ Settings related to the download process.
Additional settings passed to yt-dlp.
- **Format**: This controls which streams get downloaded and is equivalent to passing `--format` to yt-dlp. Use one of the recommended one or look at the documentation of [yt-dlp](https://github.com/yt-dlp/yt-dlp#format-selection). Please note: The option `--merge-output-format mp4` is automatically passed to yt-dlp to guarantee browser compatibility.
- **Embed Metadata**: This saves the available tags directly into the media file by passing `--embed-metadata` to yt-dlp.
- **Embed Thumbnail**: This will save the thumbnail into the media file by passing `--embed-thumbnail` to yt-dlp.
# Actions
@@ -58,3 +59,11 @@ BE AWARE: This will **not** backup any media files, just the metadata from the E
The restore functionality will expect the same zip file in *cache/backup* as created from the **Backup database** function. This will recreate the index from the snapshot. If there are multiple backup files in the folder, the newest one will take priority.
BE AWARE: This will **replace** your current index with the one from the backup file. This won't restore any media files.
## Rescan Filesystem
This function will go through all your media files and looks at the whole index to try to find any issues:
- Should the filename not match with the indexed media url, this will rename the video files correctly and update the index with the new link.
- When you delete media files from the filesystem outside of the Tube Archivist interface, this will delete leftover metadata from the index.
- When you have media files that are not indexed yet, this will grab the metadata from YouTube like it was a newly downloaded video. This can be useful when restoring from an older backup file with missing metadata but already downloaded mediafiles. NOTE: This only works if the media files are named in the same convention as Tube Archivist does, particularly the YouTube ID needs to be at the same index in the filename, alternatively see above for *Manual Media Files Import*.
BE AWARE: There is no undo.

View File

@@ -6,6 +6,14 @@ from django.apps import AppConfig
from home.src.config import AppConfig as ArchivistConfig
from home.src.helper import RedisArchivist
from home.src.index_management import index_check
from home.src.thumbnails import validate_thumbnails
def sync_redis_state():
"""make sure redis gets new config.json values"""
print("sync redis")
config_handler = ArchivistConfig()
config_handler.load_new_defaults()
def make_folders():
@@ -39,4 +47,6 @@ class HomeConfig(AppConfig):
def ready(self):
release_lock()
index_check()
sync_redis_state()
make_folders()
validate_thumbnails()

View File

@@ -18,9 +18,11 @@
"limit_speed": false,
"sleep_interval": 3,
"format": false,
"add_metadata": false
"add_metadata": false,
"add_thumbnail": false
},
"application": {
"app_root": "/app",
"cache_dir": "/cache",
"videos": "/youtube",
"file_template": "%(id)s_%(title)s.mp4",

View File

@@ -70,6 +70,7 @@ class PendingList:
all_downloaded = self.get_all_downloaded()
# loop
bulk_list = []
all_videos_added = []
for video in missing_videos:
if isinstance(video, str):
youtube_id = video
@@ -87,10 +88,12 @@ class PendingList:
video["channel_indexed"] = True
else:
video["channel_indexed"] = False
thumb_url = video["vid_thumb_url"]
video["status"] = "pending"
action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
bulk_list.append(json.dumps(action))
bulk_list.append(json.dumps(video))
all_videos_added.append((youtube_id, thumb_url))
# notify
mess_dict = {
"status": "pending",
@@ -108,6 +111,8 @@ class PendingList:
if not request.ok:
print(request)
return all_videos_added
@staticmethod
def get_youtube_details(youtube_id):
"""get details from youtubedl for single pending video"""
@@ -121,9 +126,14 @@ class PendingList:
except youtube_dl.utils.DownloadError:
print("failed to extract info for: " + youtube_id)
return False
# stop if video is streaming live now
if vid["is_live"]:
return False
# parse response
seconds = vid["duration"]
duration_str = DurationConverter.get_str(seconds)
if duration_str == "NA":
print(f"skip extracting duration for: {youtube_id}")
upload_date = vid["upload_date"]
upload_dt = datetime.strptime(upload_date, "%Y%m%d")
published = upload_dt.strftime("%Y-%m-%d")
@@ -165,12 +175,11 @@ class PendingList:
all_hits = json_data["hits"]["hits"]
if all_hits:
for hit in all_hits:
youtube_id = hit["_source"]["youtube_id"]
status = hit["_source"]["status"]
if status == "pending":
all_pending.append(hit["_source"])
elif status == "ignore":
all_ignore.append(youtube_id)
all_ignore.append(hit["_source"])
search_after = hit["sort"]
# update search_after with last hit data
data["search_after"] = search_after
@@ -342,9 +351,9 @@ class ChannelSubscription:
all_channels = self.get_channels()
pending_handler = PendingList()
all_pending, all_ignore = pending_handler.get_all_pending()
all_pending_ids = [i["youtube_id"] for i in all_pending]
all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
all_downloaded = pending_handler.get_all_downloaded()
to_ignore = all_pending_ids + all_ignore + all_downloaded
to_ignore = all_ids + all_downloaded
missing_videos = []
counter = 1
for channel in all_channels:
@@ -390,6 +399,8 @@ class ChannelSubscription:
print(request.text)
# sync to videos
channel_handler.sync_to_videos()
if channel_handler.source == "scraped":
channel_handler.get_channel_art()
def playlist_extractor(playlist_id):
@@ -444,6 +455,18 @@ class VideoDownloader:
"""add pending videos to download queue"""
all_pending, _ = PendingList().get_all_pending()
to_add = [i["youtube_id"] for i in all_pending]
if not to_add:
# there is nothing pending
print("download queue is empty")
mess_dict = {
"status": "downloading",
"level": "error",
"title": "Download queue is empty",
"message": "",
}
RedisArchivist().set_message("progress:download", mess_dict)
return
queue = RedisQueue("dl_queue")
queue.add_list(to_add)
@@ -470,8 +493,8 @@ class VideoDownloader:
}
RedisArchivist().set_message("progress:download", mess_dict)
def dl_single_vid(self, youtube_id):
"""download single video"""
def build_obs(self):
"""build obs dictionary for yt-dlp"""
obs = {
"default_search": "ytsearch",
"merge_output_format": "mp4",
@@ -482,9 +505,11 @@ class VideoDownloader:
+ self.config["application"]["file_template"]
),
"progress_hooks": [self.progress_hook],
"noprogress": True,
"quiet": True,
"continuedl": True,
"retries": 3,
"writethumbnail": False,
}
if self.config["downloads"]["format"]:
obs["format"] = self.config["downloads"]["format"]
@@ -505,15 +530,29 @@ class VideoDownloader:
}
)
if self.config["downloads"]["add_thumbnail"]:
postprocessors.append(
{
"key": "EmbedThumbnail",
"already_have_thumbnail": True,
}
)
obs["writethumbnail"] = True
obs["postprocessors"] = postprocessors
return obs
def dl_single_vid(self, youtube_id):
"""download single video"""
dl_cache = self.config["application"]["cache_dir"] + "/download/"
obs = self.build_obs()
# check if already in cache to continue from there
cache_dir = self.config["application"]["cache_dir"]
cached = os.listdir(cache_dir + "/download/")
all_cached = ignore_filelist(cached)
all_cached = ignore_filelist(os.listdir(dl_cache))
for file_name in all_cached:
if youtube_id in file_name:
obs["outtmpl"] = cache_dir + "/download/" + file_name
obs["outtmpl"] = os.path.join(dl_cache, file_name)
with youtube_dl.YoutubeDL(obs) as ydl:
try:
ydl.download([youtube_id])
@@ -522,33 +561,36 @@ class VideoDownloader:
sleep(10)
ydl.download([youtube_id])
if obs["writethumbnail"]:
# webp files don't get cleaned up automatically
all_cached = ignore_filelist(os.listdir(dl_cache))
to_clean = [i for i in all_cached if not i.endswith(".mp4")]
for file_name in to_clean:
file_path = os.path.join(dl_cache, file_name)
os.remove(file_path)
def move_to_archive(self, vid_dict):
"""move downloaded video from cache to archive"""
videos = self.config["application"]["videos"]
channel_name = vid_dict["channel"]["channel_name"]
channel_name_clean = clean_string(channel_name)
media_url = vid_dict["media_url"]
youtube_id = vid_dict["youtube_id"]
# make archive folder
videos = self.config["application"]["videos"]
new_folder = os.path.join(videos, channel_name_clean)
os.makedirs(new_folder, exist_ok=True)
host_uid = self.config["application"]["HOST_UID"]
host_gid = self.config["application"]["HOST_GID"]
channel_name = clean_string(vid_dict["channel"]["channel_name"])
# make archive folder with correct permissions
new_folder = os.path.join(videos, channel_name)
if not os.path.exists(new_folder):
os.makedirs(new_folder)
os.chown(new_folder, host_uid, host_gid)
# find real filename
cache_dir = self.config["application"]["cache_dir"]
cached = os.listdir(cache_dir + "/download/")
all_cached = ignore_filelist(cached)
all_cached = ignore_filelist(os.listdir(cache_dir + "/download/"))
for file_str in all_cached:
if youtube_id in file_str:
if vid_dict["youtube_id"] in file_str:
old_file = file_str
old_file_path = os.path.join(cache_dir, "download", old_file)
new_file_path = os.path.join(videos, media_url)
# move and fix permission
new_file_path = os.path.join(videos, vid_dict["media_url"])
# move media file and fix permission
shutil.move(old_file_path, new_file_path)
os.chown(
new_file_path,
self.config["application"]["HOST_UID"],
self.config["application"]["HOST_GID"],
)
os.chown(new_file_path, host_uid, host_gid)
def delete_from_pending(self, youtube_id):
"""delete downloaded video from pending index if its there"""

View File

@@ -237,6 +237,10 @@ class DurationConverter:
@staticmethod
def get_str(duration_sec):
"""takes duration in sec and returns clean string"""
if not duration_sec:
# failed to extract
return "NA"
hours = duration_sec // 3600
minutes = (duration_sec - (hours * 3600)) // 60
secs = duration_sec - (hours * 3600) - (minutes * 60)

View File

@@ -16,6 +16,7 @@ import yt_dlp as youtube_dl
from bs4 import BeautifulSoup
from home.src.config import AppConfig
from home.src.helper import DurationConverter, clean_string, process_url_list
from home.src.thumbnails import ThumbManager
class YoutubeChannel:
@@ -24,6 +25,7 @@ class YoutubeChannel:
CONFIG = AppConfig().config
ES_URL = CONFIG["application"]["es_url"]
CACHE_DIR = CONFIG["application"]["cache_dir"]
VIDEOS = CONFIG["application"]["videos"]
def __init__(self, channel_id):
self.channel_id = channel_id
@@ -152,6 +154,15 @@ class YoutubeChannel:
return meta_channel_dict
def get_channel_art(self):
"""download channel art for new channels"""
channel_id = self.channel_id
channel_thumb = self.channel_dict["channel_thumb_url"]
channel_banner = self.channel_dict["channel_banner_url"]
ThumbManager().download_chan(
[(channel_id, channel_thumb, channel_banner)]
)
def upload_to_es(self):
"""upload channel data to elastic search"""
url = f"{self.ES_URL}/ta_channel/_doc/{self.channel_id}"
@@ -160,16 +171,6 @@ class YoutubeChannel:
if not response.ok:
print(response.text)
def clear_cache(self):
"""delete banner and thumb from cache if available"""
channel_cache = os.path.join(self.CACHE_DIR, "channels")
thumb = os.path.join(channel_cache, self.channel_id + "_thumb.jpg")
banner = os.path.join(channel_cache, self.channel_id + "_banner.jpg")
if os.path.exists(thumb):
os.remove(thumb)
if os.path.exists(banner):
os.remove(banner)
def sync_to_videos(self):
"""sync new channel_dict to all videos of channel"""
headers = {"Content-type": "application/json"}
@@ -193,17 +194,45 @@ class YoutubeChannel:
if not request.ok:
print(request.text)
def get_total_hits(self):
"""get total channels indexed"""
def get_folder_path(self):
"""get folder where media files get stored"""
channel_name = self.channel_dict["channel_name"]
folder_name = clean_string(channel_name)
folder_path = os.path.join(self.VIDEOS, folder_name)
return folder_path
def delete_es_videos(self):
"""delete all channel documents from elasticsearch"""
headers = {"Content-type": "application/json"}
data = {"query": {"match_all": {}}}
data = {
"query": {
"term": {"channel.channel_id": {"value": self.channel_id}}
}
}
payload = json.dumps(data)
url = f"{self.ES_URL}/ta_channel/_search?filter_path=hits.total"
request = requests.post(url, data=payload, headers=headers)
if not request.ok:
print(request.text)
total_hits = json.loads(request.text)["hits"]["total"]["value"]
return total_hits
url = self.ES_URL + "/ta_video/_delete_by_query"
response = requests.post(url, data=payload, headers=headers)
if not response.ok:
print(response.text)
def delete_channel(self):
"""delete channel and all videos"""
print(f"deleting {self.channel_id} and all matching media files")
folder_path = self.get_folder_path()
print("delete all media files")
all_videos = os.listdir(folder_path)
for video in all_videos:
video_path = os.path.join(folder_path, video)
os.remove(video_path)
os.rmdir(folder_path)
ThumbManager().delete_chan_thumb(self.channel_id)
print("delete indexed videos")
self.delete_es_videos()
url = self.ES_URL + "/ta_channel/_doc/" + self.channel_id
response = requests.delete(url)
if not response.ok:
print(response.text)
class YoutubeVideo:
@@ -217,11 +246,12 @@ class YoutubeVideo:
def __init__(self, youtube_id):
self.youtube_id = youtube_id
self.channel_id = None
self.vid_dict = self.get_wrapper()
self.vid_dict = None
def get_wrapper(self):
def get_vid_dict(self):
"""wrapper to loop around youtube_dl to retry on failure"""
print(f"get video data for {self.youtube_id}")
vid_dict = False
for i in range(3):
try:
vid_dict = self.get_youtubedl_vid_data()
@@ -232,7 +262,7 @@ class YoutubeVideo:
else:
break
return vid_dict
self.vid_dict = vid_dict
def get_youtubedl_vid_data(self):
"""parse youtubedl extract info"""
@@ -343,13 +373,6 @@ class YoutubeVideo:
if not response.ok:
print(response.text)
def delete_cache(self):
"""delete thumbnail from cache if exist"""
video_cache = os.path.join(self.CACHE_DIR, "videos")
thumb = os.path.join(video_cache, self.youtube_id + ".jpg")
if os.path.exists(thumb):
os.remove(thumb)
def deactivate(self):
"""deactivate document on extractor error"""
youtube_id = self.youtube_id
@@ -362,6 +385,22 @@ class YoutubeVideo:
if not response.ok:
print(response.text)
def delete_media_file(self):
"""delete video file, meta data, thumbnails"""
# delete media file
es_vid_dict = self.get_es_data()
media_url = es_vid_dict["_source"]["media_url"]
print(f"delete {media_url} from file system")
to_delete = os.path.join(self.VIDEOS, media_url)
os.remove(to_delete)
# delete from index
url = f"{self.ES_URL}/ta_video/_doc/{self.youtube_id}"
response = requests.delete(url)
if not response.ok:
print(response.text)
# delete thumbs from cache
ThumbManager().delete_vid_thumb(self.youtube_id)
class WatchState:
"""handle watched checkbox for videos and channels"""
@@ -384,6 +423,16 @@ class WatchState:
print(f"marked {self.youtube_id} as watched")
def mark_as_unwatched(self):
"""revert watched state to false"""
url_type = self.dedect_type()
if url_type == "video":
self.mark_vid_watched(revert=True)
elif url_type == "channel":
self.mark_channel_watched(revert=True)
print(f"revert {self.youtube_id} as unwatched")
def dedect_type(self):
"""find youtube id type"""
url_process = process_url_list([self.youtube_id])
@@ -391,18 +440,21 @@ class WatchState:
return url_type
def mark_vid_watched(self):
def mark_vid_watched(self, revert=False):
"""change watched status of single video"""
url = self.ES_URL + "/ta_video/_update/" + self.youtube_id
data = {
"doc": {"player": {"watched": True, "watched_date": self.stamp}}
}
if revert:
data["doc"]["player"]["watched"] = False
payload = json.dumps(data)
request = requests.post(url, data=payload, headers=self.HEADERS)
if not request.ok:
print(request.text)
def mark_channel_watched(self):
def mark_channel_watched(self, revert=False):
"""change watched status of every video in channel"""
es_url = self.ES_URL
headers = self.HEADERS
@@ -415,6 +467,9 @@ class WatchState:
{"set": {"field": "player.watched_date", "value": self.stamp}},
],
}
if revert:
data["processors"][0]["set"]["value"] = False
payload = json.dumps(data)
url = f"{es_url}/_ingest/pipeline/{youtube_id}"
request = requests.put(url, data=payload, headers=headers)
@@ -438,6 +493,7 @@ class WatchState:
def index_new_video(youtube_id, missing_vid=False):
"""combine video and channel classes for new video index"""
vid_handler = YoutubeVideo(youtube_id)
vid_handler.get_vid_dict()
if not vid_handler.vid_dict:
raise ValueError("failed to get metadata for " + youtube_id)
@@ -452,6 +508,7 @@ def index_new_video(youtube_id, missing_vid=False):
if channel_handler.source == "scraped":
channel_handler.channel_dict["channel_subscribed"] = False
channel_handler.upload_to_es()
channel_handler.get_channel_art()
# upload video to es
vid_handler.upload_to_es()
# return vid_dict for further processing

View File

@@ -24,6 +24,7 @@ from home.src.helper import (
ignore_filelist,
)
from home.src.index import YoutubeChannel, YoutubeVideo, index_new_video
from home.src.thumbnails import ThumbManager
class Reindex:
@@ -144,6 +145,7 @@ class Reindex:
def reindex_single_video(youtube_id):
"""refresh data for single video"""
vid_handler = YoutubeVideo(youtube_id)
vid_handler.get_vid_dict()
if not vid_handler.vid_dict:
# stop if deactivated
vid_handler.deactivate()
@@ -161,7 +163,10 @@ class Reindex:
vid_handler.vid_dict["channel"] = channel_dict
# update
vid_handler.upload_to_es()
vid_handler.delete_cache()
thumb_handler = ThumbManager()
thumb_handler.delete_vid_thumb(youtube_id)
to_download = (youtube_id, vid_handler.vid_dict["vid_thumb_url"])
thumb_handler.download_vid([to_download], notify=False)
@staticmethod
def reindex_single_channel(channel_id):
@@ -172,9 +177,15 @@ class Reindex:
scrape=True
)
channel_handler.channel_dict["channel_subscribed"] = subscribed
# update
channel_handler.upload_to_es()
channel_handler.sync_to_videos()
channel_handler.clear_cache()
thumb_handler = ThumbManager()
thumb_handler.delete_chan_thumb(channel_id)
channel_thumb = channel_handler.channel_dict["channel_thumb_url"]
channel_banner = channel_handler.channel_dict["channel_banner_url"]
to_download = (channel_id, channel_thumb, channel_banner)
thumb_handler.download_chan([to_download])
def reindex(self):
"""reindex what's needed"""
@@ -297,6 +308,7 @@ class FilesystemScanner:
"""rename media files as identified by find_bad_media_url"""
for bad_filename in self.to_rename:
channel, filename, expected_filename = bad_filename
print(f"renaming [{filename}] to [{expected_filename}]")
old_path = os.path.join(self.VIDEOS, channel, filename)
new_path = os.path.join(self.VIDEOS, channel, expected_filename)
os.rename(old_path, new_path)
@@ -306,6 +318,7 @@ class FilesystemScanner:
bulk_list = []
for video_mismatch in self.mismatch:
youtube_id, media_url = video_mismatch
print(f"{youtube_id}: fixing media url {media_url}")
action = {"update": {"_id": youtube_id, "_index": "ta_video"}}
source = {"doc": {"media_url": media_url}}
bulk_list.append(json.dumps(action))
@@ -323,7 +336,8 @@ class FilesystemScanner:
def delete_from_index(self):
"""find indexed but deleted mediafile"""
for indexed in self.to_delete:
youtube_id, _ = indexed
youtube_id = indexed[0]
print(f"deleting {youtube_id} from index")
url = self.ES_URL + "/ta_video/_doc/" + youtube_id
request = requests.delete(url)
if not request.ok:
@@ -399,6 +413,8 @@ class ManualImport:
def process_import(self):
"""go through identified media files"""
all_videos_added = []
for media_file in self.identified:
json_file = media_file["json_file"]
video_file = media_file["video_file"]
@@ -411,6 +427,9 @@ class ManualImport:
# identify and archive
vid_dict = index_new_video(youtube_id)
VideoDownloader([youtube_id]).move_to_archive(vid_dict)
youtube_id = vid_dict["youtube_id"]
thumb_url = vid_dict["vid_thumb_url"]
all_videos_added.append((youtube_id, thumb_url))
# cleanup
if os.path.exists(video_path):
@@ -419,6 +438,8 @@ class ManualImport:
json_path = os.path.join(self.CACHE_DIR, "import", json_file)
os.remove(json_path)
return all_videos_added
def move_to_cache(self, video_path, youtube_id):
"""move identified video file to cache, convert to mp4"""
file_name = os.path.split(video_path)[-1]
@@ -456,12 +477,16 @@ def scan_filesystem():
filesystem_handler = FilesystemScanner()
filesystem_handler.list_comarison()
if filesystem_handler.to_rename:
print("renaming files")
filesystem_handler.rename_files()
if filesystem_handler.mismatch:
print("fixing media urls in index")
filesystem_handler.send_mismatch_bulk()
if filesystem_handler.to_delete:
print("delete metadata from index")
filesystem_handler.delete_from_index()
if filesystem_handler.to_index:
print("index new videos")
for missing_vid in filesystem_handler.to_index:
youtube_id = missing_vid[2]
index_new_video(youtube_id, missing_vid=missing_vid)

View File

@@ -7,14 +7,12 @@ Functionality:
"""
import math
import os
import urllib.parse
from datetime import datetime
import requests
from home.src.config import AppConfig
from home.src.helper import ignore_filelist
from PIL import Image
from home.src.thumbnails import ThumbManager
class SearchHandler:
@@ -23,11 +21,10 @@ class SearchHandler:
CONFIG = AppConfig().config
CACHE_DIR = CONFIG["application"]["cache_dir"]
def __init__(self, url, data, cache=True):
def __init__(self, url, data):
self.max_hits = None
self.url = url
self.data = data
self.cache = cache
def get_data(self):
"""get the data"""
@@ -61,10 +58,6 @@ class SearchHandler:
channel_dict = self.channel_cache_link(hit)
if channel_dict not in all_channels:
all_channels.append(channel_dict)
if self.cache:
# validate cache
self.cache_dl_vids(all_videos)
self.cache_dl_chan(all_channels)
return return_value
@@ -103,50 +96,6 @@ class SearchHandler:
}
return channel_dict
def cache_dl_vids(self, all_videos):
"""video thumbs links for cache"""
vid_cache = os.path.join(self.CACHE_DIR, "videos")
vid_cached = os.listdir(vid_cache)
all_vid_cached = ignore_filelist(vid_cached)
# videos
for video_dict in all_videos:
youtube_id = video_dict["youtube_id"]
if not youtube_id + ".jpg" in all_vid_cached:
cache_path = os.path.join(vid_cache, youtube_id + ".jpg")
thumb_url = video_dict["vid_thumb"]
img_raw = requests.get(thumb_url, stream=True).raw
img = Image.open(img_raw)
width, height = img.size
if not width / height == 16 / 9:
new_height = width / 16 * 9
offset = (height - new_height) / 2
img = img.crop((0, offset, width, height - offset))
img.convert("RGB").save(cache_path)
def cache_dl_chan(self, all_channels):
"""download channel thumbs"""
chan_cache = os.path.join(self.CACHE_DIR, "channels")
chan_cached = os.listdir(chan_cache)
all_chan_cached = ignore_filelist(chan_cached)
for channel_dict in all_channels:
channel_id_cache = channel_dict["channel_id"]
channel_banner_url = channel_dict["chan_banner"]
channel_banner = channel_id_cache + "_banner.jpg"
channel_thumb_url = channel_dict["chan_thumb"]
channel_thumb = channel_id_cache + "_thumb.jpg"
# thumb
if channel_thumb_url and channel_thumb not in all_chan_cached:
cache_path = os.path.join(chan_cache, channel_thumb)
img_raw = requests.get(channel_thumb_url, stream=True).content
with open(cache_path, "wb") as f:
f.write(img_raw)
# banner
if channel_banner_url and channel_banner not in all_chan_cached:
cache_path = os.path.join(chan_cache, channel_banner)
img_raw = requests.get(channel_banner_url, stream=True).content
with open(cache_path, "wb") as f:
f.write(img_raw)
@staticmethod
def hit_cleanup(hit):
"""clean up and parse data from a single hit"""
@@ -168,6 +117,11 @@ class SearchHandler:
date_str = datetime.strftime(date_refresh, "%d %b, %Y")
hit["source"]["vid_last_refresh"] = date_str
if "vid_thumb_url" in hit_keys:
youtube_id = hit["source"]["youtube_id"]
thumb_path = ThumbManager().vid_thumb_path(youtube_id)
hit["source"]["vid_thumb_url"] = thumb_path
if "channel_last_refresh" in hit_keys:
refreshed = hit["source"]["channel_last_refresh"]
date_refresh = datetime.fromtimestamp(refreshed)
@@ -208,7 +162,7 @@ class SearchForm:
}
},
}
look_up = SearchHandler(url, data, cache=False)
look_up = SearchHandler(url, data)
search_results = look_up.get_data()
return {"results": search_results}

View File

@@ -0,0 +1,243 @@
"""
functionality:
- handle download and caching for thumbnails
"""
import os
from collections import Counter
from time import sleep
import home.src.download as download
import requests
from home.src.config import AppConfig
from home.src.helper import RedisArchivist, ignore_filelist
from PIL import Image
class ThumbManager:
"""handle thumbnails related functions"""
CONFIG = AppConfig().config
CACHE_DIR = CONFIG["application"]["cache_dir"]
VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
def get_all_thumbs(self):
"""get all video artwork already downloaded"""
all_thumb_folders = ignore_filelist(os.listdir(self.VIDEO_DIR))
all_thumbs = []
for folder in all_thumb_folders:
folder_path = os.path.join(self.VIDEO_DIR, folder)
if os.path.isfile(folder_path):
self.update_path(folder)
all_thumbs.append(folder_path)
continue
# raise exemption here in a future version
# raise FileExistsError("video cache dir has files inside")
all_folder_thumbs = ignore_filelist(os.listdir(folder_path))
all_thumbs.extend(all_folder_thumbs)
return all_thumbs
def update_path(self, file_name):
"""reorganize thumbnails into folders as update path from v0.0.5"""
folder_name = file_name[0].lower()
folder_path = os.path.join(self.VIDEO_DIR, folder_name)
old_file = os.path.join(self.VIDEO_DIR, file_name)
new_file = os.path.join(folder_path, file_name)
os.makedirs(folder_path, exist_ok=True)
os.rename(old_file, new_file)
def get_needed_thumbs(self, missing_only=False):
"""get a list of all missing thumbnails"""
all_thumbs = self.get_all_thumbs()
all_indexed = download.PendingList().get_all_indexed()
all_in_queue, all_ignored = download.PendingList().get_all_pending()
needed_thumbs = []
for video in all_indexed:
youtube_id = video["_source"]["youtube_id"]
thumb_url = video["_source"]["vid_thumb_url"]
if missing_only:
if youtube_id + ".jpg" not in all_thumbs:
needed_thumbs.append((youtube_id, thumb_url))
else:
needed_thumbs.append((youtube_id, thumb_url))
for video in all_in_queue + all_ignored:
youtube_id = video["youtube_id"]
thumb_url = video["vid_thumb_url"]
if missing_only:
if youtube_id + ".jpg" not in all_thumbs:
needed_thumbs.append((youtube_id, thumb_url))
else:
needed_thumbs.append((youtube_id, thumb_url))
return needed_thumbs
def get_missing_channels(self):
"""get all channel artwork"""
all_channel_art = os.listdir(self.CHANNEL_DIR)
files = [i[0:24] for i in all_channel_art]
cached_channel_ids = [k for (k, v) in Counter(files).items() if v > 1]
channels = download.ChannelSubscription().get_channels(
subscribed_only=False
)
missing_channels = []
for channel in channels:
channel_id = channel["channel_id"]
if channel_id not in cached_channel_ids:
channel_banner = channel["channel_banner_url"]
channel_thumb = channel["channel_thumb_url"]
missing_channels.append(
(channel_id, channel_thumb, channel_banner)
)
return missing_channels
def get_raw_img(self, img_url, thumb_type):
"""get raw image from youtube and handle 404"""
try:
app_root = self.CONFIG["application"]["app_root"]
except KeyError:
# lazy keyerror fix to not have to deal with a strange startup
# racing contition between the threads in HomeConfig.ready()
app_root = "/app"
default_map = {
"video": os.path.join(
app_root, "static/img/default-video-thumb.jpg"
),
"icon": os.path.join(
app_root, "static/img/default-channel-icon.jpg"
),
"banner": os.path.join(
app_root, "static/img/default-channel-banner.jpg"
),
}
if img_url:
response = requests.get(img_url, stream=True)
if not response.ok and not response.status_code == 404:
print("retry thumbnail download for " + img_url)
sleep(5)
response = requests.get(img_url, stream=True)
else:
response = False
if not response or response.status_code == 404:
# use default
img_raw = Image.open(default_map[thumb_type])
else:
# use response
img_obj = response.raw
img_raw = Image.open(img_obj)
return img_raw
def download_vid(self, missing_thumbs, notify=True):
"""download all missing thumbnails from list"""
total = len(missing_thumbs)
print(f"downloading {total} thumbnails")
# videos
# counter as update path from v0.0.5, remove in future version
counter = 0
for youtube_id, thumb_url in missing_thumbs:
folder_name = youtube_id[0].lower()
folder_path = os.path.join(self.VIDEO_DIR, folder_name)
thumb_path_part = self.vid_thumb_path(youtube_id)
thumb_path = os.path.join(self.CACHE_DIR, thumb_path_part)
os.makedirs(folder_path, exist_ok=True)
img_raw = self.get_raw_img(thumb_url, "video")
width, height = img_raw.size
if not width / height == 16 / 9:
new_height = width / 16 * 9
offset = (height - new_height) / 2
img_raw = img_raw.crop((0, offset, width, height - offset))
img_raw.convert("RGB").save(thumb_path)
if notify:
mess_dict = {
"status": "pending",
"level": "info",
"title": "Adding to download queue.",
"message": "Downloading Thumbnails...",
}
RedisArchivist().set_message("progress:download", mess_dict)
if counter % 50 == 0 and counter != 0:
print(f"thumbnail progress: {counter}/{total}")
counter = counter + 1
def download_chan(self, missing_channels):
"""download needed artwork for channels"""
print(f"downloading {len(missing_channels)} channel artwork")
for channel in missing_channels:
channel_id, channel_thumb, channel_banner = channel
thumb_path = os.path.join(
self.CHANNEL_DIR, channel_id + "_thumb.jpg"
)
img_raw = self.get_raw_img(channel_thumb, "icon")
img_raw.convert("RGB").save(thumb_path)
banner_path = os.path.join(
self.CHANNEL_DIR, channel_id + "_banner.jpg"
)
img_raw = self.get_raw_img(channel_banner, "banner")
img_raw.convert("RGB").save(banner_path)
mess_dict = {
"status": "pending",
"level": "info",
"title": "Adding to download queue.",
"message": "Downloading Channel Art...",
}
RedisArchivist().set_message("progress:download", mess_dict)
@staticmethod
def vid_thumb_path(youtube_id):
"""build expected path for video thumbnail from youtube_id"""
folder_name = youtube_id[0].lower()
folder_path = os.path.join("videos", folder_name)
thumb_path = os.path.join(folder_path, youtube_id + ".jpg")
return thumb_path
def delete_vid_thumb(self, youtube_id):
"""delete video thumbnail if exists"""
thumb_path = self.vid_thumb_path(youtube_id)
to_delete = os.path.join(self.CACHE_DIR, thumb_path)
if os.path.exists(to_delete):
os.remove(to_delete)
def delete_chan_thumb(self, channel_id):
"""delete all artwork of channel"""
thumb = os.path.join(self.CHANNEL_DIR, channel_id + "_thumb.jpg")
banner = os.path.join(self.CHANNEL_DIR, channel_id + "_banner.jpg")
if os.path.exists(thumb):
os.remove(thumb)
if os.path.exists(banner):
os.remove(banner)
def cleanup_downloaded(self):
"""find downloaded thumbnails without video indexed"""
all_thumbs = self.get_all_thumbs()
all_indexed = self.get_needed_thumbs()
all_needed_thumbs = [i[0] + ".jpg" for i in all_indexed]
for thumb in all_thumbs:
if thumb not in all_needed_thumbs:
# cleanup
youtube_id = thumb.rstrip(".jpg")
self.delete_vid_thumb(youtube_id)
def validate_thumbnails():
"""check if all thumbnails are there and organized correctly"""
handler = ThumbManager()
thumbs_to_download = handler.get_needed_thumbs(missing_only=True)
handler.download_vid(thumbs_to_download)
missing_channels = handler.get_missing_channels()
handler.download_chan(missing_channels)
handler.cleanup_downloaded()

View File

@@ -11,7 +11,12 @@ from home.src.config import AppConfig
from home.src.download import ChannelSubscription, PendingList, VideoDownloader
from home.src.helper import RedisArchivist, RedisQueue
from home.src.index_management import backup_all_indexes, restore_from_backup
from home.src.reindex import ManualImport, reindex_old_documents
from home.src.reindex import (
ManualImport,
reindex_old_documents,
scan_filesystem,
)
from home.src.thumbnails import ThumbManager
CONFIG = AppConfig().config
REDIS_HOST = os.environ.get("REDIS_HOST")
@@ -33,7 +38,8 @@ def update_subscribed():
missing_videos = channel_handler.find_missing()
if missing_videos:
pending_handler = PendingList()
pending_handler.add_to_pending(missing_videos)
all_videos_added = pending_handler.add_to_pending(missing_videos)
ThumbManager().download_vid(all_videos_added)
# check if reindex is needed
check_reindex.delay()
@@ -41,7 +47,6 @@ def update_subscribed():
@shared_task
def download_pending():
"""download latest pending videos"""
have_lock = False
my_lock = RedisArchivist().get_lock("downloading")
@@ -62,11 +67,9 @@ def download_pending():
@shared_task
def download_single(youtube_id):
"""start download single video now"""
queue = RedisQueue("dl_queue")
queue.add_priority(youtube_id)
print("Added to queue with priority: " + youtube_id)
# start queue if needed
have_lock = False
my_lock = RedisArchivist().get_lock("downloading")
@@ -89,7 +92,8 @@ def extrac_dl(youtube_ids):
"""parse list passed and add to pending"""
pending_handler = PendingList()
missing_videos = pending_handler.parse_url_list(youtube_ids)
pending_handler.add_to_pending(missing_videos)
all_videos_added = pending_handler.add_to_pending(missing_videos)
ThumbManager().download_vid(all_videos_added)
@shared_task
@@ -101,7 +105,6 @@ def check_reindex():
@shared_task
def run_manual_import():
"""called from settings page, to go through import folder"""
print("starting media file import")
have_lock = False
my_lock = RedisArchivist().get_lock("manual_import")
@@ -111,7 +114,8 @@ def run_manual_import():
if have_lock:
import_handler = ManualImport()
if import_handler.identified:
import_handler.process_import()
all_videos_added = import_handler.process_import()
ThumbManager().download_vid(all_videos_added)
else:
print("Did not acquire lock form import.")
@@ -136,7 +140,9 @@ def run_restore_backup():
def kill_dl(task_id):
"""kill download worker task by ID"""
app.control.revoke(task_id, terminate=True)
if task_id:
app.control.revoke(task_id, terminate=True)
_ = RedisArchivist().del_message("dl_queue_id")
RedisQueue("dl_queue").clear()
@@ -154,3 +160,9 @@ def kill_dl(task_id):
"message": "",
}
RedisArchivist().set_message("progress:download", mess_dict)
@shared_task
def rescan_filesystem():
"""check the media folder for mismatches"""
scan_filesystem()

View File

@@ -96,7 +96,7 @@
</div>
<div class="footer">
<div class="boxed-content">
<span>© 2021 The Tube Archivist v0.0.5 | <a href="https://github.com/bbilly1/tubearchivist" target="_blank">Github</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a></span>
<span>© 2021 The Tube Archivist v0.0.6 | <a href="https://github.com/bbilly1/tubearchivist" target="_blank">Github</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a></span>
</div>
</div>
</body>

View File

@@ -54,13 +54,11 @@
{% if channels %}
{% for channel in channels %}
<div class="channel-item {{ view_style }}">
{% if channel.source.channel_banner_url %}
<div class="channel-banner {{ view_style }}">
<a href="{% url 'channel_id' channel.source.channel_id %}">
<img src="/cache/channels/{{ channel.source.channel_id }}_banner.jpg" alt="{{ channel.source.channel_id }}-banner">
</a>
</div>
{% endif %}
<div class="channel-banner {{ view_style }}">
<a href="{% url 'channel_id' channel.source.channel_id %}">
<img src="/cache/channels/{{ channel.source.channel_id }}_banner.jpg" alt="{{ channel.source.channel_id }}-banner">
</a>
</div>
<div class="info-box info-box-2 {{ view_style }}">
<div class="info-box-item">
<div class="round-img">

View File

@@ -6,9 +6,7 @@
<h1>Channel: {{ channel_info.channel_name }}</h1>
</div>
<div class="channel-banner">
{% if channel_info.channel_banner_url %}
<a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
{% endif %}
<a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
</div>
<div class="info-box info-box-3">
<div class="info-box-item">
@@ -38,20 +36,24 @@
</p>
</div>
</div>
{% if videos %}
<div class="info-box-item">
<div>
{% if channel_info.channel_views >= 1000000 %}
<p>Channel views: {{ channel_info.channel_views|intword }}</p>
{% else %}
{% elif channel_info.channel_views > 0 %}
<p>Channel views: {{ channel_info.channel_views|intcomma }}</p>
{% endif %}
<p>Total Videos archived: {{ max_hits }}</p>
<p>Watched: <button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="{{ channel_info.channel_id }}" onclick="isWatched(this.id)">Mark as watched</button></p>
{% if max_hits %}
<p>Total Videos archived: {{ max_hits }}</p>
<p>Watched: <button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="{{ channel_info.channel_id }}" onclick="isWatched(this.id)">Mark as watched</button></p>
{% endif %}
<p>Channel id: {{ channel_info.channel_id }}</p>
<button onclick="deleteConfirm()" id="delete-item">Delete Channel</button>
<div class="delete-confirm" id="delete-button">
<span>Delete {{ channel_info.channel_name }} including all videos? </span><button class="danger-button" onclick="deleteChannel(this)" data-id="{{ channel_info.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
</div>
</div>
</div>
{% endif %}
</div>
{% if channel_info.channel_description %}
<div class="info-box-item description-box">
@@ -71,10 +73,10 @@
{% if videos %}
{% for video in videos %}
<div class="video-item {{ view_style }}">
<a href="#player" data-src="/media/{{ video.source.media_url }}" data-thumb="/cache/videos/{{ video.source.youtube_id }}.jpg" data-title="{{ video.source.title }}" data-channel="{{ video.source.channel.channel_name }}" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
<a href="#player" data-src="/media/{{ video.source.media_url }}" data-thumb="/cache/{{ video.source.vid_thumb_url }}" data-title="{{ video.source.title }}" data-channel="{{ video.source.channel.channel_name }}" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
<div class="video-thumb-wrap {{ view_style }}">
<div class="video-thumb">
<img src="/cache/videos/{{ video.source.youtube_id }}.jpg" alt="video-thumb">
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
</div>
<div class="video-play">
<img src="{% static 'img/icon-play.svg' %}" alt="play-icon">
@@ -84,7 +86,7 @@
<div class="video-desc {{ view_style }}">
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
{% if video.source.player.watched %}
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" class="seen-icon">
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
{% else %}
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.source.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
{% endif %}

View File

@@ -57,7 +57,7 @@
{% for video in all_video_hits %}
<div class="dl-item {{ view_style }}" id="dl-{{ video.youtube_id }}">
<div class="dl-thumb {{ view_style }}">
<img src="{{ video.vid_thumb_url }}" alt="video_thumb">
<img src="/cache/{{ video.vid_thumb_url }}" alt="video_thumb">
</div>
<div class="dl-desc {{ view_style }}">
{% if show_ignored_only %}

View File

@@ -48,10 +48,10 @@
{% if videos %}
{% for video in videos %}
<div class="video-item {{ view_style }}">
<a href="#player" data-src="/media/{{ video.source.media_url }}" data-thumb="/cache/videos/{{ video.source.youtube_id }}.jpg" data-title="{{ video.source.title }}" data-channel="{{ video.source.channel.channel_name }}" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
<a href="#player" data-src="/media/{{ video.source.media_url }}" data-thumb="/cache/{{ video.source.vid_thumb_url }}" data-title="{{ video.source.title }}" data-channel="{{ video.source.channel.channel_name }}" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
<div class="video-thumb-wrap {{ view_style }}">
<div class="video-thumb">
<img src="/cache/videos/{{ video.source.youtube_id }}.jpg" alt="video-thumb">
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
</div>
<div class="video-play">
<img src="{% static 'img/icon-play.svg' %}" alt="play-icon">
@@ -61,7 +61,7 @@
<div class="video-desc {{ view_style }}">
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
{% if video.source.player.watched %}
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" class="seen-icon">
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.source.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
{% else %}
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.source.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
{% endif %}

View File

@@ -88,6 +88,15 @@
<option value="1">embed metadata</option>
</select>
</div>
<div class="settings-item">
<p>Current thumbnail embed setting: <span class="settings-current">{{ config.downloads.add_thumbnail }}</span></p>
<i>Embed thumbnail into the mediafile.</i><br>
<select name="downloads.add_thumbnail" id="downloads.add_thumbnail"">
<option value="" disabled selected> -- change thumbnail embed -- </option>
<option value="0">don't embed thumbnail</option>
<option value="1">embed thumbnail</option>
</select>
</div>
</div>
<button type="submit">Update Settings</button>
</form>
@@ -96,7 +105,7 @@
</div>
<div class="settings-group">
<h2>Manual media files import.</h2>
<p>Add files to the <span class="settings-current">cache/import</span> folder. Make sure to follow the instructions on <a href="https://github.com/bbilly1/tubearchivist#import-your-existing-library" target="_blank">Github</a>.</p>
<p>Add files to the <span class="settings-current">cache/import</span> folder. Make sure to follow the instructions in the Github <a href="https://github.com/bbilly1/tubearchivist/wiki/Settings" target="_blank">Wiki</a>.</p>
<div id="manual-import">
<button onclick="manualImport()">Start import</button>
</div>
@@ -117,7 +126,11 @@
</div>
</div>
<div class="settings-group">
<p>Rescan filesystem.</p>
<i>Coming soon</i>
<h2>Rescan filesystem</h2>
<p><span class="danger-zone">Danger Zone</span>: This will delete the metadata of deleted videos from the filesystem.</p>
<p>Rescan your media folder looking for missing videos and clean up index. More infos on the Github <a href="https://github.com/bbilly1/tubearchivist/wiki/Settings" target="_blank">Wiki</a>.</p>
<div id="fs-rescan">
<button onclick="fsRescan()">Rescan filesystem</button>
</div>
</div>
{% endblock content %}

View File

@@ -8,7 +8,7 @@
<div class="video-main">
<video
src="/media/{{ video.media_url }}"
poster="/cache/videos/{{ video.youtube_id }}.jpg" controls preload="false"
poster="/cache/{{ video.vid_thumb_url }}" controls preload="false"
type='video/mp4' width="100%">
</video>
</div>
@@ -34,7 +34,7 @@
<p>Last refreshed: {{ video.vid_last_refresh }}</p>
<p class="video-info-watched">Watched:
{% if video.player.watched %}
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" class="seen-icon" id="{{ video.youtube_id }}">
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
{% else %}
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
{% endif %}
@@ -45,6 +45,10 @@
<p>Youtube: Deactivated</p>
{% endif %}
<p>Video ID: {{ video.youtube_id }}</p>
<button onclick="deleteConfirm()" id="delete-item">Delete Video</button>
<div class="delete-confirm" id="delete-button">
<span>Are you sure? </span><button class="danger-button" onclick="deleteVideo(this)" data-id="{{ video.youtube_id }}" data-redirect = "{{ video.channel.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
</div>
</div>
</div>
<div class="info-box-item">
@@ -52,7 +56,13 @@
<p>Views: {{ video.stats.view_count|intcomma }}</p>
<p class="thumb-icon like"><img src="{% static 'img/icon-thumb.svg' %}" alt="thumbs-up">: {{ video.stats.like_count|intcomma }}</p>
<p class="thumb-icon dislike"><img src="{% static 'img/icon-thumb.svg' %}" alt="thumbs-down">: {{ video.stats.dislike_count|intcomma }}</p>
<p>Rating: {{ video.stats.average_rating }}</p>
{% if video.stats.average_rating %}
<p class="rating-stars">Rating:
{% for star in video.stats.average_rating %}
<img src="/static/img/icon-star-{{ star }}.svg" alt="{{ star }}">
{% endfor %}
</p>
{% endif %}
</div>
</div>
</div>

View File

@@ -1,3 +0,0 @@
from django.test import TestCase # noqa: F401 - Unused import
# Create your tests here.

View File

View File

@@ -0,0 +1,42 @@
"""All test classes"""
from django.test import TestCase
class URLTests(TestCase):
"""test if all expected URL are there"""
def test_home_view(self):
"""check homepage"""
response = self.client.get("/")
self.assertEqual(response.status_code, 200)
def test_about_view(self):
"""check about page"""
response = self.client.get("/about/")
self.assertEqual(response.status_code, 200)
def test_downloads_view(self):
"""check downloads page"""
response = self.client.get("/downloads/")
self.assertEqual(response.status_code, 200)
def test_channel_view(self):
"""check channel page"""
response = self.client.get("/channel/")
self.assertEqual(response.status_code, 200)
def test_settings_view(self):
"""check settings page"""
response = self.client.get("/settings/")
self.assertEqual(response.status_code, 200)
def test_progress_view(self):
"""check ajax progress endpoint"""
response = self.client.get("/downloads/progress/")
self.assertEqual(response.status_code, 200)
def test_process_view(self):
"""check process ajax endpoint"""
response = self.client.get("/process/")
self.assertEqual(response.status_code, 200)

View File

@@ -19,7 +19,7 @@ urlpatterns = [
path("downloads/", DownloadView.as_view(), name="downloads"),
path("settings/", SettingsView.as_view(), name="settings"),
path("process/", views.process, name="process"),
path("downloads/progress", views.progress, name="progress"),
path("downloads/progress/", views.progress, name="progress"),
path("channel/", ChannelView.as_view(), name="channel"),
path(
"channel/<slug:channel_id_detail>/",

View File

@@ -15,13 +15,14 @@ from django.views import View
from home.src.config import AppConfig
from home.src.download import ChannelSubscription, PendingList
from home.src.helper import RedisArchivist, RedisQueue, process_url_list
from home.src.index import WatchState
from home.src.index import WatchState, YoutubeChannel, YoutubeVideo
from home.src.searching import Pagination, SearchForm, SearchHandler
from home.tasks import (
download_pending,
download_single,
extrac_dl,
kill_dl,
rescan_filesystem,
run_backup,
run_manual_import,
run_restore_backup,
@@ -157,7 +158,7 @@ class DownloadView(View):
url = config["application"]["es_url"] + "/ta_download/_search"
data = self.build_data(pagination_handler, show_ignored_only)
search = SearchHandler(url, data, cache=False)
search = SearchHandler(url, data)
videos_hits = search.get_data()
max_hits = search.max_hits
@@ -407,6 +408,12 @@ class VideoView(View):
look_up = SearchHandler(url, data)
video_hit = look_up.get_data()
video_data = video_hit[0]["source"]
try:
rating = video_data["stats"]["average_rating"]
video_data["stats"]["average_rating"] = self.star_creator(rating)
except KeyError:
video_data["stats"]["average_rating"] = False
video_title = video_data["title"]
context = {"video": video_data, "title": video_title, "colors": colors}
return render(request, "home/video.html", context)
@@ -419,6 +426,20 @@ class VideoView(View):
colors = config["application"]["colors"]
return es_url, colors
@staticmethod
def star_creator(rating):
"""convert rating float to stars"""
stars = []
for _ in range(1, 6):
if rating >= 0.75:
stars.append("full")
elif 0.25 < rating < 0.75:
stars.append("half")
else:
stars.append("empty")
rating = rating - 1
return stars
class SettingsView(View):
"""resolves to /settings/
@@ -489,6 +510,7 @@ class PostData:
"""map dict key and return function to execute"""
exec_map = {
"watched": self.watched,
"un_watched": self.un_watched,
"change_view": self.change_view,
"rescan_pending": self.rescan_pending,
"ignore": self.ignore,
@@ -505,7 +527,10 @@ class PostData:
"manual-import": self.manual_import,
"db-backup": self.db_backup,
"db-restore": self.db_restore,
"fs-rescan": self.fs_rescan,
"channel-search": self.channel_search,
"delete-video": self.delete_video,
"delete-channel": self.delete_channel,
}
return exec_map[self.to_exec]
@@ -515,6 +540,11 @@ class PostData:
WatchState(self.exec_val).mark_as_watched()
return {"success": True}
def un_watched(self):
"""mark as unwatched"""
WatchState(self.exec_val).mark_as_unwatched()
return {"success": True}
def change_view(self):
"""process view changes in home, channel, and downloads"""
origin, new_view = self.exec_val.split(":")
@@ -558,7 +588,10 @@ class PostData:
RedisQueue("dl_queue").clear()
elif to_execute == "kill":
task_id = RedisArchivist().get_message("dl_queue_id")
print("brutally killing " + task_id)
if not isinstance(task_id, str):
task_id = False
else:
print("brutally killing " + task_id)
kill_dl(task_id)
return {"success": True}
@@ -652,9 +685,28 @@ class PostData:
run_restore_backup.delay()
return {"success": True}
@staticmethod
def fs_rescan():
"""start file system rescan task"""
print("start filesystem scan")
rescan_filesystem.delay()
return {"success": True}
def channel_search(self):
"""search for channel name as_you_type"""
search_query = self.exec_val
print("searching for: " + search_query)
search_results = SearchForm().search_channels(search_query)
return search_results
def delete_video(self):
"""delete media file, metadata and thumb"""
youtube_id = self.exec_val
YoutubeVideo(youtube_id).delete_media_file()
return {"success": True}
def delete_channel(self):
"""delete channel and all matching videos"""
channel_id = self.exec_val
YoutubeChannel(channel_id).delete_channel()
return {"success": True}

View File

@@ -1,9 +1,9 @@
beautifulsoup4==4.10.0
celery==5.1.2
Django==3.2.7
Pillow==8.3.2
Django==3.2.8
Pillow==8.4.0
redis==3.5.3
requests==2.26.0
uWSGI==2.0.19.1
uWSGI==2.0.20
whitenoise==5.3.0
yt_dlp==2021.9.25
yt_dlp==2021.10.10

View File

@@ -2,6 +2,7 @@
--main-bg: #00202f;
--highlight-bg: #00293b;
--highlight-error: #990202;
--highlight-error-light: #c44343;
--highlight-bg-transparent: #00293baf;
--main-font: #eeeeee;
--accent-font-dark: #259485;

View File

@@ -1,11 +1,12 @@
:root {
--main-bg: #eeeeee;
--highlight-bg: #d9e0d9;
--highlight-error: #f79c9c;
--highlight-error: #990202;
--highlight-error-light: #c44343;
--highlight-bg-transparent: #00293baf;
--main-font: #00202f;
--accent-font-dark: #259485;
--accent-font-light: #35b399;
--img-filter: invert(50%) sepia(9%) saturate(2940%) hue-rotate(122deg) brightness(94%) contrast(90%);
--img-filter-error: invert(83%) sepia(35%) saturate(1238%) hue-rotate(297deg) brightness(103%) contrast(97%);
--img-filter-error: invert(16%) sepia(60%) saturate(3717%) hue-rotate(349deg) brightness(86%) contrast(120%);
}

View File

@@ -1,12 +1,12 @@
@font-face {
font-family: 'Sen-Bold';
src: url('../font/Sen-Bold.ttf.woff');
src: url('../font/Sen-Bold.woff');
font-family: 'Sen-Bold';
}
@font-face {
font-family: 'Sen-Regular';
src: url('../font/Sen-Regular.ttf.woff');
src: url('../font/Sen-Regular.woff');
font-family: 'Sen-Regular';
}
@@ -205,6 +205,19 @@ button:hover {
color: var(--main-font);
}
/* delete button */
.delete-confirm {
display: none;
}
.danger-button {
background-color: var(--highlight-error);
}
.danger-button:hover {
background-color: var(--highlight-error-light);
}
/* navigation */
.top-nav {
display: block;
@@ -440,15 +453,12 @@ button:hover {
}
.unseen-icon,
.seen-icon,
.close-button {
cursor: pointer;
filter: var(--img-filter);
}
.seen-icon {
filter: var(--img-filter);
}
.video-more {
text-decoration: underline;
text-align: right;
@@ -518,7 +528,8 @@ button:hover {
margin-left: 5px;
}
.thumb-icon img {
.thumb-icon img,
.rating-stars img {
width: 20px;
margin: 0;
filter: var(--img-filter);
@@ -661,6 +672,10 @@ button:hover {
background-color: var(--highlight-error);
}
.download-progress.error h3 {
color: #fff;
}
/* settings */
.settings-group {
background-color: var(--highlight-bg);
@@ -678,6 +693,7 @@ button:hover {
.danger-zone {
background-color: var(--highlight-error);
color: #fff;
padding: 3px;
}

View File

@@ -0,0 +1,94 @@
Copyright (c) 2015, Kosal Sen, Philatype (<http://philatype.com>),
with Reserved Font Name Sen.
This Font Software is licensed under the SIL Open Font License, Version 1.1.
This license is copied below, and is also available with a FAQ at:
http://scripts.sil.org/OFL
-----------------------------------------------------------
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
-----------------------------------------------------------
PREAMBLE
The goals of the Open Font License (OFL) are to stimulate worldwide
development of collaborative font projects, to support the font creation
efforts of academic and linguistic communities, and to provide a free and
open framework in which fonts may be shared and improved in partnership
with others.
The OFL allows the licensed fonts to be used, studied, modified and
redistributed freely as long as they are not sold by themselves. The
fonts, including any derivative works, can be bundled, embedded,
redistributed and/or sold with any software provided that any reserved
names are not used by derivative works. The fonts and derivatives,
however, cannot be released under any other type of license. The
requirement for fonts to remain under this license does not apply
to any document created using the fonts or their derivatives.
DEFINITIONS
"Font Software" refers to the set of files released by the Copyright
Holder(s) under this license and clearly marked as such. This may
include source files, build scripts and documentation.
"Reserved Font Name" refers to any names specified as such after the
copyright statement(s).
"Original Version" refers to the collection of Font Software components as
distributed by the Copyright Holder(s).
"Modified Version" refers to any derivative made by adding to, deleting,
or substituting -- in part or in whole -- any of the components of the
Original Version, by changing formats or by porting the Font Software to a
new environment.
"Author" refers to any designer, engineer, programmer, technical
writer or other person who contributed to the Font Software.
PERMISSION & CONDITIONS
Permission is hereby granted, free of charge, to any person obtaining
a copy of the Font Software, to use, study, copy, merge, embed, modify,
redistribute, and sell modified and unmodified copies of the Font
Software, subject to the following conditions:
1) Neither the Font Software nor any of its individual components,
in Original or Modified Versions, may be sold by itself.
2) Original or Modified Versions of the Font Software may be bundled,
redistributed and/or sold with any software, provided that each copy
contains the above copyright notice and this license. These can be
included either as stand-alone text files, human-readable headers or
in the appropriate machine-readable metadata fields within text or
binary files as long as those fields can be easily viewed by the user.
3) No Modified Version of the Font Software may use the Reserved Font
Name(s) unless explicit written permission is granted by the corresponding
Copyright Holder. This restriction only applies to the primary font name as
presented to the users.
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
Software shall not be used to promote, endorse or advertise any
Modified Version, except to acknowledge the contribution(s) of the
Copyright Holder(s) and the Author(s) or with their explicit written
permission.
5) The Font Software, modified or unmodified, in part or in whole,
must be distributed entirely under this license, and must not be
distributed under any other license. The requirement for fonts to
remain under this license does not apply to any document created
using the Font Software.
TERMINATION
This license becomes null and void if any of the above conditions are
not met.
DISCLAIMER
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
OTHER DEALINGS IN THE FONT SOFTWARE.

Binary file not shown.

Binary file not shown.

Binary file not shown.

After

Width:  |  Height:  |  Size: 61 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 84 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 56 KiB

View File

@@ -0,0 +1,65 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
width="500"
height="500"
viewBox="0 0 132.29197 132.29167"
version="1.1"
id="svg1303"
inkscape:version="1.1.1 (3bf5ae0d25, 2021-09-20)"
sodipodi:docname="icon-star-empty.svg"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns="http://www.w3.org/2000/svg"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:dc="http://purl.org/dc/elements/1.1/">
<defs
id="defs1297" />
<sodipodi:namedview
id="base"
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1.0"
inkscape:pageopacity="0.0"
inkscape:pageshadow="2"
inkscape:zoom="2.4005076"
inkscape:cx="40.40812"
inkscape:cy="211.20533"
inkscape:document-units="mm"
inkscape:current-layer="layer1"
showgrid="false"
units="px"
inkscape:window-width="3840"
inkscape:window-height="2112"
inkscape:window-x="0"
inkscape:window-y="48"
inkscape:window-maximized="1"
inkscape:pagecheckerboard="0" />
<metadata
id="metadata1300">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title />
</cc:Work>
</rdf:RDF>
</metadata>
<g
inkscape:label="Ebene 1"
inkscape:groupmode="layer"
id="layer1"
transform="translate(0,-164.70764)">
<path
id="path1255"
style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
inkscape:transform-center-y="-6.4703827"
d="M 250 21.158203 L 189.78906 194.37891 L 6.4375 198.11523 L 152.57617 308.9082 L 99.470703 484.43945 L 250 379.69141 L 400.5293 484.43945 L 347.42383 308.9082 L 493.5625 198.11523 L 310.21094 194.37891 L 250 21.158203 z M 250 96.408203 L 291.69141 216.34766 L 418.64453 218.93555 L 317.45703 295.65039 L 354.22852 417.18945 L 250 344.66211 L 145.77148 417.18945 L 182.54297 295.65039 L 81.355469 218.93555 L 208.30859 216.34766 L 250 96.408203 z "
transform="matrix(0.26458394,0,0,0.26458394,0,164.70749)" />
</g>
</svg>

After

Width:  |  Height:  |  Size: 2.5 KiB

View File

@@ -0,0 +1,74 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
width="500"
height="500"
viewBox="0 0 132.29197 132.29167"
version="1.1"
id="svg1303"
inkscape:version="0.92.4 (5da689c313, 2019-01-14)"
sodipodi:docname="Icons_star.svg">
<defs
id="defs1297" />
<sodipodi:namedview
id="base"
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1.0"
inkscape:pageopacity="0.0"
inkscape:pageshadow="2"
inkscape:zoom="1.0105705"
inkscape:cx="61.891881"
inkscape:cy="148.25167"
inkscape:document-units="mm"
inkscape:current-layer="layer1"
showgrid="false"
units="px"
inkscape:window-width="1920"
inkscape:window-height="1017"
inkscape:window-x="-8"
inkscape:window-y="-8"
inkscape:window-maximized="1" />
<metadata
id="metadata1300">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title />
</cc:Work>
</rdf:RDF>
</metadata>
<g
inkscape:label="Ebene 1"
inkscape:groupmode="layer"
id="layer1"
transform="translate(0,-164.70764)">
<path
sodipodi:type="star"
style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
id="path1255"
sodipodi:sides="5"
sodipodi:cx="66.145981"
sodipodi:cy="237.32387"
sodipodi:r1="67.758904"
sodipodi:r2="27.103561"
sodipodi:arg1="-1.5707963"
sodipodi:arg2="-0.9424778"
inkscape:flatsided="false"
inkscape:rounded="-3.46945e-18"
inkscape:randomized="0"
d="m 66.145983,169.56496 15.931071,45.83167 48.511476,0.98859 L 91.923,245.69933 105.97366,292.14197 66.145981,264.42743 26.318295,292.14197 40.368962,245.69933 1.7034347,216.38521 50.214907,215.39663 Z"
inkscape:transform-center-y="-6.4703827" />
</g>
</svg>

After

Width:  |  Height:  |  Size: 2.5 KiB

View File

@@ -0,0 +1,66 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
width="500"
height="500"
viewBox="0 0 132.29197 132.29167"
version="1.1"
id="svg1303"
inkscape:version="1.1.1 (3bf5ae0d25, 2021-09-20)"
sodipodi:docname="icon-star-half.svg"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns="http://www.w3.org/2000/svg"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:dc="http://purl.org/dc/elements/1.1/">
<defs
id="defs1297" />
<sodipodi:namedview
id="base"
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1.0"
inkscape:pageopacity="0.0"
inkscape:pageshadow="2"
inkscape:zoom="1.0105705"
inkscape:cx="44.034533"
inkscape:cy="168.22181"
inkscape:document-units="mm"
inkscape:current-layer="layer1"
showgrid="false"
units="px"
inkscape:window-width="3840"
inkscape:window-height="2078"
inkscape:window-x="0"
inkscape:window-y="82"
inkscape:window-maximized="1"
inkscape:pagecheckerboard="0" />
<metadata
id="metadata1300">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title />
</cc:Work>
</rdf:RDF>
</metadata>
<g
inkscape:label="Ebene 1"
inkscape:groupmode="layer"
id="layer1"
transform="translate(0,-164.70764)">
<path
id="path1255"
style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:markers fill stroke"
inkscape:transform-center-x="1.1220981e-06"
inkscape:transform-center-y="-6.4703827"
d="M 250 21.158203 L 189.78906 194.37891 L 6.4375 198.11523 L 152.57617 308.9082 L 99.470703 484.43945 L 250 379.69141 L 400.5293 484.43945 L 347.42383 308.9082 L 493.5625 198.11523 L 310.21094 194.37891 L 250 21.158203 z M 250 96.408203 L 291.69141 216.34766 L 418.64453 218.93555 L 317.45703 295.65039 L 354.22852 417.18945 L 250 344.66211 L 250 96.408203 z "
transform="matrix(0.26458394,0,0,0.26458394,0,164.70749)" />
</g>
</svg>

After

Width:  |  Height:  |  Size: 2.4 KiB

View File

@@ -32,7 +32,7 @@ function handleInterval() {
var watchDownload = setInterval(function() {
var req = new XMLHttpRequest();
req.responseType = 'json';
req.open('GET', '/downloads/progress', true);
req.open('GET', '/downloads/progress/', true);
req.onload = function() {
var dlProgress = req.response;
if (dlProgress['status']) {

View File

@@ -15,10 +15,25 @@ function isWatched(youtube_id) {
seenIcon.setAttribute('src', "/static/img/icon-seen.svg");
seenIcon.setAttribute('alt', 'seen-icon');
seenIcon.setAttribute('id', youtube_id);
seenIcon.setAttribute('title', "Mark as unwatched");
seenIcon.setAttribute('onclick', "isUnwatched(this.id)");
seenIcon.classList = 'seen-icon';
document.getElementById(youtube_id).replaceWith(seenIcon);
}
function isUnwatched(youtube_id) {
var payload = JSON.stringify({'un_watched': youtube_id});
sendPost(payload);
var unseenIcon = document.createElement('img');
unseenIcon.setAttribute('src', "/static/img/icon-unseen.svg");
unseenIcon.setAttribute('alt', 'unseen-icon');
unseenIcon.setAttribute('id', youtube_id);
unseenIcon.setAttribute('title', "Mark as watched");
unseenIcon.setAttribute('onclick', "isWatched(this.id)");
unseenIcon.classList = 'unseen-icon';
document.getElementById(youtube_id).replaceWith(unseenIcon);
}
function unsubscribe(channel_id) {
var payload = JSON.stringify({'unsubscribe': channel_id});
sendPost(payload);
@@ -141,7 +156,7 @@ function dbBackup() {
function dbRestore() {
var payload = JSON.stringify({'db-restore': true});
sendPost(payload)
sendPost(payload);
// clear button
var message = document.createElement('p');
message.innerText = 'restoring from backup';
@@ -150,6 +165,51 @@ function dbRestore() {
toReplace.appendChild(message);
}
function fsRescan() {
var payload = JSON.stringify({'fs-rescan': true});
sendPost(payload);
// clear button
var message = document.createElement('p');
message.innerText = 'File system scan in progress';
var toReplace = document.getElementById('fs-rescan');
toReplace.innerHTML = '';
toReplace.appendChild(message);
}
// delete from file system
function deleteConfirm() {
to_show = document.getElementById("delete-button");
document.getElementById("delete-item").style.display = 'none';
to_show.style.display = "block";
}
function deleteVideo(button) {
var to_delete = button.getAttribute("data-id");
var to_redirect = button.getAttribute("data-redirect");
var payload = JSON.stringify({"delete-video": to_delete});
sendPost(payload);
setTimeout(function(){
var redirect = "/channel/" + to_redirect;
window.location.replace(redirect);
return false;
}, 1000);
}
function deleteChannel(button) {
var to_delete = button.getAttribute("data-id");
var payload = JSON.stringify({"delete-channel": to_delete});
sendPost(payload);
setTimeout(function(){
window.location.replace("/channel/");
return false;
}, 1000);
}
function cancelDelete() {
document.getElementById("delete-button").style.display = 'none';
document.getElementById("delete-item").style.display = 'block';
}
// player
function createPlayer(button) {
var mediaUrl = button.getAttribute('data-src');