mirrors/metube
1
0
Fork 0
mirror of https://github.com/alexta69/metube.git synced 2025-04-03 20:27:36 +03:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse source
This commit is contained in:
rdiaz738 2025-03-01 14:07:31 -08:00
commit 4acb48b12a
13 changed files with 3613 additions and 2244 deletions
Download patch file Download diff file Expand all files Collapse all files

2
app/dl_formats.py
View file

@ -35,7 +35,7 @@ def get_format(format: str, quality: str) -> str:
return "bestaudio/best"
# video {res} {vfmt} + audio {afmt} {res} {vfmt}
vfmt, afmt = ("[ext=mp4]", "[ext=m4a]") if format == "mp4" else ("", "")
vres = f"[height<={quality}]" if quality not in ("best", "best_ios") else ""
vres = f"[height<={quality}]" if quality not in ("best", "best_ios", "worst") else ""
vcombo = vres + vfmt
if quality == "best_ios":

62
app/main.py
View file

@ -4,6 +4,8 @@
import os
import sys
from aiohttp import web
import ssl
import socket
import socketio
import logging
import json
@ -28,17 +30,24 @@ class Config:
'PUBLIC_HOST_AUDIO_URL': 'audio_download/',
'OUTPUT_TEMPLATE': '%(title)s.%(ext)s',
'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)s %(section_title)s.%(ext)s',
'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s',
'DEFAULT_OPTION_PLAYLIST_STRICT_MODE' : 'false',
'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0',
'YTDL_OPTIONS': '{}',
'YTDL_OPTIONS_FILE': '',
'ROBOTS_TXT': '',
'HOST': '0.0.0.0',
'PORT': '8081',
'HTTPS': 'false',
'CERTFILE': '',
'KEYFILE': '',
'BASE_DIR': '',
'DEFAULT_THEME': 'auto',
'DOWNLOAD_MODE': 'concurrent', # Can be 'sequential', 'concurrent', or 'limited'
'MAX_CONCURRENT_DOWNLOADS': 3, # Used if DOWNLOAD_MODE is 'limited'
}
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN')
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS')
def __init__(self):
for k, v in self._DEFAULTS.items():
@ -129,13 +138,22 @@ async def add(request):
format = post.get('format')
folder = post.get('folder')
custom_name_prefix = post.get('custom_name_prefix')
playlist_strict_mode = post.get('playlist_strict_mode')
playlist_item_limit = post.get('playlist_item_limit')
auto_start = post.get('auto_start')
if custom_name_prefix is None:
custom_name_prefix = ''
if auto_start is None:
auto_start = True
status = await dqueue.add(url, quality, format, folder, custom_name_prefix, auto_start)
log.info(f"Download added to queue: {url}")
if playlist_strict_mode is None:
playlist_strict_mode = config.DEFAULT_OPTION_PLAYLIST_STRICT_MODE
if playlist_item_limit is None:
playlist_item_limit = config.DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT
playlist_item_limit = int(playlist_item_limit)
status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start)
return web.Response(text=serializer.encode(status))
@routes.post(config.URL_PREFIX + 'delete')
@ -160,12 +178,14 @@ async def start(request):
@routes.get(config.URL_PREFIX + 'history')
async def history(request):
history = { 'done': [], 'queue': []}
history = { 'done': [], 'queue': [], 'pending': []}
for _, v in dqueue.queue.saved_items():
history['queue'].append(v)
for _, v in dqueue.done.saved_items():
history['done'].append(v)
for _ ,v in dqueue.pending.saved_items():
history['pending'].append(v)
log.info("Sending download history")
return web.Response(text=serializer.encode(history))
@ -216,6 +236,16 @@ def index(request):
response.set_cookie('metube_theme', config.DEFAULT_THEME)
return response
@routes.get(config.URL_PREFIX + 'robots.txt')
def robots(request):
if config.ROBOTS_TXT:
response = web.FileResponse(os.path.join(config.BASE_DIR, config.ROBOTS_TXT))
else:
response = web.Response(
text="User-agent: *\nDisallow: /download/\nDisallow: /audio_download/\n"
)
return response
if config.URL_PREFIX != '/':
@routes.get('/')
def index_redirect_root(request):
@ -248,15 +278,23 @@ async def on_prepare(request, response):
response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
app.on_response_prepare.append(on_prepare)
def supports_reuse_port():
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
sock.close()
return True
except (AttributeError, OSError):
return False
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
log.info(f"Listening on {config.HOST}:{config.PORT}")
try:
if sys.platform.startswith('win'):
web.run_app(app, host=config.HOST, port=int(config.PORT))
else:
web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=True)
except Exception as e:
log.error(f"Failed to start the server: {str(e)}")
sys.exit(1)
if config.HTTPS:
ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
ssl_context.load_cert_chain(certfile=config.CERTFILE, keyfile=config.KEYFILE)
web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port(), ssl_context=ssl_context)
else:
web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port())

57
app/ytdl.py
View file

@ -7,6 +7,8 @@ import asyncio
import multiprocessing
import logging
import re
import yt_dlp.networking.impersonate
from dl_formats import get_format, get_opts, AUDIO_FORMATS
from datetime import datetime
@ -225,7 +227,7 @@ class DownloadQueue:
async def __import_queue(self):
for k, v in self.queue.saved_items():
await self.add(v.url, v.quality, v.format, v.folder, v.custom_name_prefix)
await self.add(v.url, v.quality, v.format, v.folder, v.custom_name_prefix, v.playlist_strict_mode, v.playlist_item_limit)
async def initialize(self):
log.info("Initializing DownloadQueue")
@ -274,13 +276,15 @@ class DownloadQueue:
self.done.put(download)
asyncio.create_task(self.notifier.completed(download.info))
def __extract_info(self, url):
def __extract_info(self, url, playlist_strict_mode):
return yt_dlp.YoutubeDL(params={
'quiet': True,
'no_color': True,
'extract_flat': True,
'ignore_no_formats_error': True,
'noplaylist': playlist_strict_mode,
'paths': {"home": self.config.DOWNLOAD_DIR, "temp": self.config.TEMP_DIR},
**({'impersonate': yt_dlp.networking.impersonate.ImpersonateTarget.from_str(self.config.YTDL_OPTIONS['impersonate'])} if 'impersonate' in self.config.YTDL_OPTIONS else {}),
**self.config.YTDL_OPTIONS,
}).extract_info(url, download=False)
@ -301,7 +305,7 @@ class DownloadQueue:
dldirectory = base_directory
return dldirectory, None
async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, auto_start, already):
async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already):
if not entry:
return {'status': 'error', 'msg': "Invalid/empty data was given."}
@ -314,46 +318,65 @@ class DownloadQueue:
error = entry["msg"]
etype = entry.get('_type') or 'video'
if etype == 'playlist':
if etype.startswith('url'):
log.debug('Processing as an url')
return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already)
elif etype == 'playlist':
log.debug('Processing as a playlist')
entries = entry['entries']
log.info(f'playlist detected with {len(entries)} entries')
playlist_index_digits = len(str(len(entries)))
results = []
if playlist_item_limit > 0:
log.info(f'Playlist item limit is set. Processing only first {playlist_item_limit} entries')
entries = entries[:playlist_item_limit]
for index, etr in enumerate(entries, start=1):
etr["_type"] = "video" # Prevents video to be treated as url and lose below properties during processing
etr["playlist"] = entry["id"]
etr["playlist_index"] = '{{0:0{0:d}d}}'.format(playlist_index_digits).format(index)
for property in ("id", "title", "uploader", "uploader_id"):
if property in entry:
etr[f"playlist_{property}"] = entry[property]
results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, auto_start, already))
results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already))
if any(res['status'] == 'error' for res in results):
return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}
return {'status': 'ok'}
elif etype == 'video' or etype.startswith('url') and 'id' in entry and 'title' in entry:
log.debug('Processing as a video')
if not self.queue.exists(entry['id']):
dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format, folder, custom_name_prefix, error)
dl = DownloadInfo(entry['id'], entry.get('title') or entry['id'], entry.get('webpage_url') or entry['url'], quality, format, folder, custom_name_prefix, error)
dldirectory, error_message = self.__calc_download_path(quality, format, folder)
if error_message is not None:
return error_message
output = self.config.OUTPUT_TEMPLATE if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{self.config.OUTPUT_TEMPLATE}'
output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER
for property, value in entry.items():
if property.startswith("playlist"):
output = output.replace(f"%({property})s", str(value))
if 'playlist' in entry and entry['playlist'] is not None:
if len(self.config.OUTPUT_TEMPLATE_PLAYLIST):
output = self.config.OUTPUT_TEMPLATE_PLAYLIST
for property, value in entry.items():
if property.startswith("playlist"):
output = output.replace(f"%({property})s", str(value))
ytdl_options = dict(self.config.YTDL_OPTIONS)
if playlist_item_limit > 0:
log.info(f'playlist limit is set. Processing only first {playlist_item_limit} entries')
ytdl_options['playlistend'] = playlist_item_limit
if auto_start is True:
download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, quality, format, self.config.YTDL_OPTIONS, dl)
download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, quality, format, ytdl_options, dl)
self.queue.put(download)
asyncio.create_task(self.__start_download(download))
else:
self.pending.put(Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, quality, format, self.config.YTDL_OPTIONS, dl))
self.pending.put(Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, quality, format, ytdl_options, dl))
await self.notifier.added(dl)
return {'status': 'ok'}
elif etype.startswith('url'):
return await self.add(entry['url'], quality, format, folder, custom_name_prefix, auto_start, already)
return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'}
async def add(self, url, quality, format, folder, custom_name_prefix, auto_start=True, already=None):
log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=}')
async def add(self, url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start=True, already=None):
log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_strict_mode=} {playlist_item_limit=}')
already = set() if already is None else already
if url in already:
log.info('recursion detected, skipping')
@ -361,10 +384,10 @@ class DownloadQueue:
else:
already.add(url)
try:
entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url)
entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url, playlist_strict_mode)
except yt_dlp.utils.YoutubeDLError as exc:
return {'status': 'error', 'msg': str(exc)}
return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, auto_start, already)
return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already)
async def start_pending(self, ids):
for id in ids: