mirror of
https://git.vectorsigma.ru/public/tubearchivist.git
synced 2026-08-04 22:49:31 +00:00
renamed django app folder to backend
This commit is contained in:
Simon
0
backend/stats/__init__.py
Normal file
0
backend/stats/__init__.py
Normal file
0
backend/stats/migrations/__init__.py
Normal file
0
backend/stats/migrations/__init__.py
Normal file
0
backend/stats/src/__init__.py
Normal file
0
backend/stats/src/__init__.py
Normal file
351
backend/stats/src/aggs.py
Normal file
351
backend/stats/src/aggs.py
Normal file
@@ -0,0 +1,351 @@
|
||||
"""aggregations"""
|
||||
|
||||
from common.src.env_settings import EnvironmentSettings
|
||||
from common.src.es_connect import ElasticWrap
|
||||
from common.src.helper import get_duration_str
|
||||
|
||||
|
||||
class AggBase:
|
||||
"""base class for aggregation calls"""
|
||||
|
||||
path: str = ""
|
||||
data: dict = {}
|
||||
name: str = ""
|
||||
|
||||
def get(self):
|
||||
"""make get call"""
|
||||
response, _ = ElasticWrap(self.path).get(self.data)
|
||||
print(f"[agg][{self.name}] took {response.get('took')} ms to process")
|
||||
|
||||
return response.get("aggregations")
|
||||
|
||||
def process(self):
|
||||
"""implement in subclassess"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Video(AggBase):
|
||||
"""get video stats"""
|
||||
|
||||
name = "video_stats"
|
||||
path = "ta_video/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"video_type": {
|
||||
"terms": {"field": "vid_type"},
|
||||
"aggs": {
|
||||
"media_size": {"sum": {"field": "media_size"}},
|
||||
"duration": {"sum": {"field": "player.duration"}},
|
||||
},
|
||||
},
|
||||
"video_active": {
|
||||
"terms": {"field": "active"},
|
||||
"aggs": {
|
||||
"media_size": {"sum": {"field": "media_size"}},
|
||||
"duration": {"sum": {"field": "player.duration"}},
|
||||
},
|
||||
},
|
||||
"video_media_size": {"sum": {"field": "media_size"}},
|
||||
"video_count": {"value_count": {"field": "youtube_id"}},
|
||||
"duration": {"sum": {"field": "player.duration"}},
|
||||
},
|
||||
}
|
||||
|
||||
def process(self):
|
||||
"""process aggregation"""
|
||||
aggregations = self.get()
|
||||
|
||||
duration = int(aggregations["duration"]["value"])
|
||||
response = {
|
||||
"doc_count": aggregations["video_count"]["value"],
|
||||
"media_size": int(aggregations["video_media_size"]["value"]),
|
||||
"duration": duration,
|
||||
"duration_str": get_duration_str(duration),
|
||||
}
|
||||
for bucket in aggregations["video_type"]["buckets"]:
|
||||
duration = int(bucket["duration"].get("value"))
|
||||
response.update(
|
||||
{
|
||||
f"type_{bucket['key']}": {
|
||||
"doc_count": bucket.get("doc_count"),
|
||||
"media_size": int(bucket["media_size"].get("value")),
|
||||
"duration": duration,
|
||||
"duration_str": get_duration_str(duration),
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
for bucket in aggregations["video_active"]["buckets"]:
|
||||
duration = int(bucket["duration"].get("value"))
|
||||
response.update(
|
||||
{
|
||||
f"active_{bucket['key_as_string']}": {
|
||||
"doc_count": bucket.get("doc_count"),
|
||||
"media_size": int(bucket["media_size"].get("value")),
|
||||
"duration": duration,
|
||||
"duration_str": get_duration_str(duration),
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
class Channel(AggBase):
|
||||
"""get channel stats"""
|
||||
|
||||
name = "channel_stats"
|
||||
path = "ta_channel/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"channel_count": {"value_count": {"field": "channel_id"}},
|
||||
"channel_active": {"terms": {"field": "channel_active"}},
|
||||
"channel_subscribed": {"terms": {"field": "channel_subscribed"}},
|
||||
},
|
||||
}
|
||||
|
||||
def process(self):
|
||||
"""process aggregation"""
|
||||
aggregations = self.get()
|
||||
|
||||
response = {
|
||||
"doc_count": aggregations["channel_count"].get("value"),
|
||||
}
|
||||
for bucket in aggregations["channel_active"]["buckets"]:
|
||||
key = f"active_{bucket['key_as_string']}"
|
||||
response.update({key: bucket.get("doc_count")})
|
||||
for bucket in aggregations["channel_subscribed"]["buckets"]:
|
||||
key = f"subscribed_{bucket['key_as_string']}"
|
||||
response.update({key: bucket.get("doc_count")})
|
||||
|
||||
return response
|
||||
|
||||
|
||||
class Playlist(AggBase):
|
||||
"""get playlist stats"""
|
||||
|
||||
name = "playlist_stats"
|
||||
path = "ta_playlist/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"playlist_count": {"value_count": {"field": "playlist_id"}},
|
||||
"playlist_active": {"terms": {"field": "playlist_active"}},
|
||||
"playlist_subscribed": {"terms": {"field": "playlist_subscribed"}},
|
||||
},
|
||||
}
|
||||
|
||||
def process(self):
|
||||
"""process aggregation"""
|
||||
aggregations = self.get()
|
||||
response = {"doc_count": aggregations["playlist_count"].get("value")}
|
||||
for bucket in aggregations["playlist_active"]["buckets"]:
|
||||
key = f"active_{bucket['key_as_string']}"
|
||||
response.update({key: bucket.get("doc_count")})
|
||||
for bucket in aggregations["playlist_subscribed"]["buckets"]:
|
||||
key = f"subscribed_{bucket['key_as_string']}"
|
||||
response.update({key: bucket.get("doc_count")})
|
||||
|
||||
return response
|
||||
|
||||
|
||||
class Download(AggBase):
|
||||
"""get downloads queue stats"""
|
||||
|
||||
name = "download_queue_stats"
|
||||
path = "ta_download/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"status": {"terms": {"field": "status"}},
|
||||
"video_type": {
|
||||
"filter": {"term": {"status": "pending"}},
|
||||
"aggs": {"type_pending": {"terms": {"field": "vid_type"}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def process(self):
|
||||
"""process aggregation"""
|
||||
aggregations = self.get()
|
||||
response = {}
|
||||
for bucket in aggregations["status"]["buckets"]:
|
||||
response.update({bucket["key"]: bucket.get("doc_count")})
|
||||
|
||||
for bucket in aggregations["video_type"]["type_pending"]["buckets"]:
|
||||
key = f"pending_{bucket['key']}"
|
||||
response.update({key: bucket.get("doc_count")})
|
||||
|
||||
return response
|
||||
|
||||
|
||||
class WatchProgress(AggBase):
|
||||
"""get watch progress"""
|
||||
|
||||
name = "watch_progress"
|
||||
path = "ta_video/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
name: {
|
||||
"terms": {"field": "player.watched"},
|
||||
"aggs": {
|
||||
"watch_docs": {
|
||||
"filter": {"terms": {"player.watched": [True, False]}},
|
||||
"aggs": {
|
||||
"true_count": {"value_count": {"field": "_index"}},
|
||||
"duration": {"sum": {"field": "player.duration"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"total_duration": {"sum": {"field": "player.duration"}},
|
||||
"total_vids": {"value_count": {"field": "_index"}},
|
||||
},
|
||||
}
|
||||
|
||||
def process(self):
|
||||
"""make the call"""
|
||||
aggregations = self.get()
|
||||
buckets = aggregations[self.name]["buckets"]
|
||||
|
||||
response = {}
|
||||
all_duration = int(aggregations["total_duration"].get("value"))
|
||||
response.update(
|
||||
{
|
||||
"total": {
|
||||
"duration": all_duration,
|
||||
"duration_str": get_duration_str(all_duration),
|
||||
"items": aggregations["total_vids"].get("value"),
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
for bucket in buckets:
|
||||
response.update(self._build_bucket(bucket, all_duration))
|
||||
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def _build_bucket(bucket, all_duration):
|
||||
"""parse bucket"""
|
||||
|
||||
duration = int(bucket["watch_docs"]["duration"]["value"])
|
||||
duration_str = get_duration_str(duration)
|
||||
items = bucket["watch_docs"]["true_count"]["value"]
|
||||
if bucket["key_as_string"] == "false":
|
||||
key = "unwatched"
|
||||
else:
|
||||
key = "watched"
|
||||
|
||||
bucket_parsed = {
|
||||
key: {
|
||||
"duration": duration,
|
||||
"duration_str": duration_str,
|
||||
"progress": duration / all_duration if all_duration else 0,
|
||||
"items": items,
|
||||
}
|
||||
}
|
||||
|
||||
return bucket_parsed
|
||||
|
||||
|
||||
class DownloadHist(AggBase):
|
||||
"""get downloads histogram last week"""
|
||||
|
||||
name = "videos_last_week"
|
||||
path = "ta_video/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
name: {
|
||||
"date_histogram": {
|
||||
"field": "date_downloaded",
|
||||
"calendar_interval": "day",
|
||||
"format": "yyyy-MM-dd",
|
||||
"order": {"_key": "desc"},
|
||||
"time_zone": EnvironmentSettings.TZ,
|
||||
},
|
||||
"aggs": {
|
||||
"total_videos": {"value_count": {"field": "youtube_id"}},
|
||||
"media_size": {"sum": {"field": "media_size"}},
|
||||
},
|
||||
}
|
||||
},
|
||||
"query": {
|
||||
"range": {
|
||||
"date_downloaded": {
|
||||
"gte": "now-7d/d",
|
||||
"time_zone": EnvironmentSettings.TZ,
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def process(self):
|
||||
"""process query"""
|
||||
aggregations = self.get()
|
||||
buckets = aggregations[self.name]["buckets"]
|
||||
|
||||
response = [
|
||||
{
|
||||
"date": i.get("key_as_string"),
|
||||
"count": i.get("doc_count"),
|
||||
"media_size": i["media_size"].get("value"),
|
||||
}
|
||||
for i in buckets
|
||||
]
|
||||
|
||||
return response
|
||||
|
||||
|
||||
class BiggestChannel(AggBase):
|
||||
"""get channel aggregations"""
|
||||
|
||||
def __init__(self, order):
|
||||
self.data["aggs"][self.name]["multi_terms"]["order"] = {order: "desc"}
|
||||
|
||||
name = "channel_stats"
|
||||
path = "ta_video/_search"
|
||||
data = {
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
name: {
|
||||
"multi_terms": {
|
||||
"terms": [
|
||||
{"field": "channel.channel_name.keyword"},
|
||||
{"field": "channel.channel_id"},
|
||||
],
|
||||
"order": {"doc_count": "desc"},
|
||||
},
|
||||
"aggs": {
|
||||
"doc_count": {"value_count": {"field": "_index"}},
|
||||
"duration": {"sum": {"field": "player.duration"}},
|
||||
"media_size": {"sum": {"field": "media_size"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
order_choices = ["doc_count", "duration", "media_size"]
|
||||
|
||||
def process(self):
|
||||
"""process aggregation, order_by validated in the view"""
|
||||
|
||||
aggregations = self.get()
|
||||
buckets = aggregations[self.name]["buckets"]
|
||||
|
||||
response = [
|
||||
{
|
||||
"id": i["key"][1],
|
||||
"name": i["key"][0].title(),
|
||||
"doc_count": i["doc_count"]["value"],
|
||||
"duration": i["duration"]["value"],
|
||||
"duration_str": get_duration_str(int(i["duration"]["value"])),
|
||||
"media_size": i["media_size"]["value"],
|
||||
}
|
||||
for i in buckets
|
||||
]
|
||||
|
||||
return response
|
||||
42
backend/stats/urls.py
Normal file
42
backend/stats/urls.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""all stats API urls"""
|
||||
|
||||
from django.urls import path
|
||||
from stats import views
|
||||
|
||||
urlpatterns = [
|
||||
path(
|
||||
"video/",
|
||||
views.StatVideoView.as_view(),
|
||||
name="api-stats-video",
|
||||
),
|
||||
path(
|
||||
"channel/",
|
||||
views.StatChannelView.as_view(),
|
||||
name="api-stats-channel",
|
||||
),
|
||||
path(
|
||||
"playlist/",
|
||||
views.StatPlaylistView.as_view(),
|
||||
name="api-stats-playlist",
|
||||
),
|
||||
path(
|
||||
"download/",
|
||||
views.StatDownloadView.as_view(),
|
||||
name="api-stats-download",
|
||||
),
|
||||
path(
|
||||
"watch/",
|
||||
views.StatWatchProgress.as_view(),
|
||||
name="api-stats-watch",
|
||||
),
|
||||
path(
|
||||
"downloadhist/",
|
||||
views.StatDownloadHist.as_view(),
|
||||
name="api-stats-downloadhist",
|
||||
),
|
||||
path(
|
||||
"biggestchannels/",
|
||||
views.StatBiggestChannel.as_view(),
|
||||
name="api-stats-biggestchannels",
|
||||
),
|
||||
]
|
||||
104
backend/stats/views.py
Normal file
104
backend/stats/views.py
Normal file
@@ -0,0 +1,104 @@
|
||||
"""all stats API views"""
|
||||
|
||||
from common.views_base import ApiBaseView
|
||||
from rest_framework.response import Response
|
||||
from stats.src.aggs import (
|
||||
BiggestChannel,
|
||||
Channel,
|
||||
Download,
|
||||
DownloadHist,
|
||||
Playlist,
|
||||
Video,
|
||||
WatchProgress,
|
||||
)
|
||||
|
||||
|
||||
class StatVideoView(ApiBaseView):
|
||||
"""resolves to /api/stats/video/
|
||||
GET: return video stats
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""get stats"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
return Response(Video().process())
|
||||
|
||||
|
||||
class StatChannelView(ApiBaseView):
|
||||
"""resolves to /api/stats/channel/
|
||||
GET: return channel stats
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""get stats"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
return Response(Channel().process())
|
||||
|
||||
|
||||
class StatPlaylistView(ApiBaseView):
|
||||
"""resolves to /api/stats/playlist/
|
||||
GET: return playlist stats
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""get stats"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
return Response(Playlist().process())
|
||||
|
||||
|
||||
class StatDownloadView(ApiBaseView):
|
||||
"""resolves to /api/stats/download/
|
||||
GET: return download stats
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""get stats"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
return Response(Download().process())
|
||||
|
||||
|
||||
class StatWatchProgress(ApiBaseView):
|
||||
"""resolves to /api/stats/watchprogress/
|
||||
GET: return watch/unwatch progress stats
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""handle get request"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
return Response(WatchProgress().process())
|
||||
|
||||
|
||||
class StatDownloadHist(ApiBaseView):
|
||||
"""resolves to /api/stats/downloadhist/
|
||||
GET: return download video count histogram for last days
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""handle get request"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
return Response(DownloadHist().process())
|
||||
|
||||
|
||||
class StatBiggestChannel(ApiBaseView):
|
||||
"""resolves to /api/stats/biggestchannels/
|
||||
GET: return biggest channels
|
||||
param: order
|
||||
"""
|
||||
|
||||
order_choices = ["doc_count", "duration", "media_size"]
|
||||
|
||||
def get(self, request):
|
||||
"""handle get request"""
|
||||
|
||||
order = request.GET.get("order", "doc_count")
|
||||
if order and order not in self.order_choices:
|
||||
message = {"message": f"invalid order parameter {order}"}
|
||||
return Response(message, status=400)
|
||||
|
||||
return Response(BiggestChannel(order).process())
|
||||
Reference in New Issue
Block a user