From 8c994bc45f7880980731d899a2c6d20faf033c66 Mon Sep 17 00:00:00 2001 From: Abdulmohsen Date: Sun, 13 Aug 2023 19:13:55 +0300 Subject: [PATCH 1/5] Updated YTDL_OPTIONS description in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9439253..ee8e115 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Certain values can be set via environment variables, using the `-e` parameter on * __URL_PREFIX__: base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`. * __OUTPUT_TEMPLATE__: the template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`. * __OUTPUT_TEMPLATE_CHAPTER__: the template for the filenames of the downloaded videos, when split into chapters via postprocessors. Defaults to `%(title)s - %(section_number)s %(section_title)s.%(ext)s`. -* __YTDL_OPTIONS__: Additional options to pass to youtube-dl, in JSON format. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L183). They roughly correspond to command-line options, though some do not have exact equivalents here, for example `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. +* __YTDL_OPTIONS__: Additional options to pass to youtube-dl, This can be path to a file or or JSON format string. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L183). They roughly correspond to command-line options, though some do not have exact equivalents here, for example `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. The following example value for `YTDL_OPTIONS` embeds English subtitles and chapter markers (for videos that have them), and also changes the permissions on the downloaded video and sets the file modification timestamp to the date of when it was downloaded: From a1e143a5a6be6ea19eaeae5bd91dc26a88e25c56 Mon Sep 17 00:00:00 2001 From: Abdulmohsen Date: Sun, 13 Aug 2023 19:14:58 +0300 Subject: [PATCH 2/5] Added support for loading YTDL_OPTIONS from file. --- app/main.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/app/main.py b/app/main.py index 6b277b3..292f069 100644 --- a/app/main.py +++ b/app/main.py @@ -3,6 +3,7 @@ import os import sys +import traceback from aiohttp import web import socketio import logging @@ -12,7 +13,9 @@ import pathlib from ytdl import DownloadQueueNotifier, DownloadQueue log = logging.getLogger('main') - +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + class Config: _DEFAULTS = { 'DOWNLOAD_DIR': '.', @@ -38,6 +41,7 @@ class Config: def __init__(self): for k, v in self._DEFAULTS.items(): setattr(self, k, os.environ[k] if k in os.environ else v) + for k, v in self.__dict__.items(): if v.startswith('%%'): setattr(self, k, getattr(self, v[2:])) @@ -46,13 +50,24 @@ class Config: log.error(f'Environment variable "{k}" is set to a non-boolean value "{v}"') sys.exit(1) setattr(self, k, v in ('true', 'True', 'on', '1')) + if not self.URL_PREFIX.endswith('/'): self.URL_PREFIX += '/' + try: - self.YTDL_OPTIONS = json.loads(self.YTDL_OPTIONS) + if isinstance(self.YTDL_OPTIONS, str) and os.path.exists(self.YTDL_OPTIONS): + log.info(f"Loading yt-dlp custom options from {self.YTDL_OPTIONS}") + with open(self.YTDL_OPTIONS) as json_data: + self.YTDL_OPTIONS = json.load(json_data) + else: + log.info(f"Loading yt-dlp custom options from Environment variable.") + self.YTDL_OPTIONS = json.loads(self.YTDL_OPTIONS) + assert isinstance(self.YTDL_OPTIONS, dict) - except (json.decoder.JSONDecodeError, AssertionError): - log.error('YTDL_OPTIONS is invalid') + if len(self.YTDL_OPTIONS) != 0: + log.info(f"Using custom yt-dlp options:\n{json.dumps(self.YTDL_OPTIONS, indent=2, ensure_ascii=False)}") + except (json.decoder.JSONDecodeError, AssertionError) as e: + log.error(f"Unable to parse YTDL_OPTIONS value. {str(e)}") sys.exit(1) config = Config() @@ -140,13 +155,13 @@ def get_custom_dirs(): dirs = list(filter(None, map(convert, path.glob('**')))) return dirs - + download_dir = recursive_dirs(config.DOWNLOAD_DIR) audio_download_dir = download_dir if config.DOWNLOAD_DIR != config.AUDIO_DOWNLOAD_DIR: audio_download_dir = recursive_dirs(config.AUDIO_DOWNLOAD_DIR) - + return { "download_dir": download_dir, "audio_download_dir": audio_download_dir @@ -195,6 +210,5 @@ app.on_response_prepare.append(on_prepare) if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) log.info(f"Listening on {config.HOST}:{config.PORT}") web.run_app(app, host=config.HOST, port=config.PORT, reuse_port=True) From 01aafe3854653ca1f0c43210a0160f3ddb184870 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Mon, 14 Aug 2023 04:53:16 +0300 Subject: [PATCH 3/5] Fixed typo and removed unused import --- README.md | 2 +- app/main.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ee8e115..9255477 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Certain values can be set via environment variables, using the `-e` parameter on * __URL_PREFIX__: base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`. * __OUTPUT_TEMPLATE__: the template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`. * __OUTPUT_TEMPLATE_CHAPTER__: the template for the filenames of the downloaded videos, when split into chapters via postprocessors. Defaults to `%(title)s - %(section_number)s %(section_title)s.%(ext)s`. -* __YTDL_OPTIONS__: Additional options to pass to youtube-dl, This can be path to a file or or JSON format string. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L183). They roughly correspond to command-line options, though some do not have exact equivalents here, for example `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. +* __YTDL_OPTIONS__: Additional options to pass to youtube-dl, This can be path to a file or JSON formatted string. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L183). They roughly correspond to command-line options, though some do not have exact equivalents here, for example `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. The following example value for `YTDL_OPTIONS` embeds English subtitles and chapter markers (for videos that have them), and also changes the permissions on the downloaded video and sets the file modification timestamp to the date of when it was downloaded: diff --git a/app/main.py b/app/main.py index 292f069..0a5d371 100644 --- a/app/main.py +++ b/app/main.py @@ -3,7 +3,6 @@ import os import sys -import traceback from aiohttp import web import socketio import logging @@ -15,7 +14,7 @@ from ytdl import DownloadQueueNotifier, DownloadQueue log = logging.getLogger('main') if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) - + class Config: _DEFAULTS = { 'DOWNLOAD_DIR': '.', From 207f8fe916772420c20f72c3e961673941b229ac Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Mon, 14 Aug 2023 05:04:39 +0300 Subject: [PATCH 4/5] switched logging to DEBUG instead of info. --- app/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 0a5d371..8d526a8 100644 --- a/app/main.py +++ b/app/main.py @@ -39,6 +39,8 @@ class Config: def __init__(self): for k, v in self._DEFAULTS.items(): + if k in os.environ: + log.debug(f"ENV override for {k} = {os.environ[k]}") setattr(self, k, os.environ[k] if k in os.environ else v) for k, v in self.__dict__.items(): @@ -64,7 +66,7 @@ class Config: assert isinstance(self.YTDL_OPTIONS, dict) if len(self.YTDL_OPTIONS) != 0: - log.info(f"Using custom yt-dlp options:\n{json.dumps(self.YTDL_OPTIONS, indent=2, ensure_ascii=False)}") + log.debug(f"Using custom yt-dlp options:\n{json.dumps(self.YTDL_OPTIONS, indent=2, ensure_ascii=False)}") except (json.decoder.JSONDecodeError, AssertionError) as e: log.error(f"Unable to parse YTDL_OPTIONS value. {str(e)}") sys.exit(1) From 27def1f67039abbbec9b931b13ddf5e05e59656c Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 19 Aug 2023 21:01:45 +0000 Subject: [PATCH 5/5] separated YTDL_OPTIONS_FILE --- README.md | 3 ++- app/main.py | 35 +++++++++++++++++++---------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 9255477..b089a46 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ Certain values can be set via environment variables, using the `-e` parameter on * __URL_PREFIX__: base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`. * __OUTPUT_TEMPLATE__: the template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`. * __OUTPUT_TEMPLATE_CHAPTER__: the template for the filenames of the downloaded videos, when split into chapters via postprocessors. Defaults to `%(title)s - %(section_number)s %(section_title)s.%(ext)s`. -* __YTDL_OPTIONS__: Additional options to pass to youtube-dl, This can be path to a file or JSON formatted string. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L183). They roughly correspond to command-line options, though some do not have exact equivalents here, for example `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. +* __YTDL_OPTIONS__: Additional options to pass to youtube-dl, in JSON format. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L183). They roughly correspond to command-line options, though some do not have exact equivalents here, for example `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. +* __YTDL_OPTIONS_FILE__: A path to a JSON file that will be loaded and used for populating `YTDL_OPTIONS` above. Please note that if both `YTDL_OPTIONS_FILE` and `YTDL_OPTIONS` are specified, the options in `YTDL_OPTIONS` take precedence. The following example value for `YTDL_OPTIONS` embeds English subtitles and chapter markers (for videos that have them), and also changes the permissions on the downloaded video and sets the file modification timestamp to the date of when it was downloaded: diff --git a/app/main.py b/app/main.py index 8d526a8..88b3d5b 100644 --- a/app/main.py +++ b/app/main.py @@ -12,8 +12,6 @@ import pathlib from ytdl import DownloadQueueNotifier, DownloadQueue log = logging.getLogger('main') -if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) class Config: _DEFAULTS = { @@ -29,6 +27,7 @@ class Config: 'OUTPUT_TEMPLATE': '%(title)s.%(ext)s', 'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)s %(section_title)s.%(ext)s', 'YTDL_OPTIONS': '{}', + 'YTDL_OPTIONS_FILE': '', 'HOST': '0.0.0.0', 'PORT': '8081', 'BASE_DIR': '', @@ -39,8 +38,6 @@ class Config: def __init__(self): for k, v in self._DEFAULTS.items(): - if k in os.environ: - log.debug(f"ENV override for {k} = {os.environ[k]}") setattr(self, k, os.environ[k] if k in os.environ else v) for k, v in self.__dict__.items(): @@ -56,21 +53,26 @@ class Config: self.URL_PREFIX += '/' try: - if isinstance(self.YTDL_OPTIONS, str) and os.path.exists(self.YTDL_OPTIONS): - log.info(f"Loading yt-dlp custom options from {self.YTDL_OPTIONS}") - with open(self.YTDL_OPTIONS) as json_data: - self.YTDL_OPTIONS = json.load(json_data) - else: - log.info(f"Loading yt-dlp custom options from Environment variable.") - self.YTDL_OPTIONS = json.loads(self.YTDL_OPTIONS) - + self.YTDL_OPTIONS = json.loads(self.YTDL_OPTIONS) assert isinstance(self.YTDL_OPTIONS, dict) - if len(self.YTDL_OPTIONS) != 0: - log.debug(f"Using custom yt-dlp options:\n{json.dumps(self.YTDL_OPTIONS, indent=2, ensure_ascii=False)}") - except (json.decoder.JSONDecodeError, AssertionError) as e: - log.error(f"Unable to parse YTDL_OPTIONS value. {str(e)}") + except (json.decoder.JSONDecodeError, AssertionError): + log.error('YTDL_OPTIONS is invalid') sys.exit(1) + if self.YTDL_OPTIONS_FILE: + log.info(f'Loading yt-dlp custom options from "{self.YTDL_OPTIONS_FILE}"') + if not os.path.exists(self.YTDL_OPTIONS_FILE): + log.error(f'File "{self.YTDL_OPTIONS_FILE}" not found') + sys.exit(1) + try: + with open(self.YTDL_OPTIONS_FILE) as json_data: + opts = json.load(json_data) + assert isinstance(opts, dict) + except (json.decoder.JSONDecodeError, AssertionError): + log.error('YTDL_OPTIONS_FILE contents is invalid') + sys.exit(1) + self.YTDL_OPTIONS.update(opts) + config = Config() class ObjectSerializer(json.JSONEncoder): @@ -211,5 +213,6 @@ app.on_response_prepare.append(on_prepare) if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) log.info(f"Listening on {config.HOST}:{config.PORT}") web.run_app(app, host=config.HOST, port=config.PORT, reuse_port=True)