diff --git a/jwb-index b/jwb-index index 8516422..e9f81dd 100755 --- a/jwb-index +++ b/jwb-index @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -from jwlib import msg -from jwlib.arguments import ArgumentParser +from jwlib.arguments import ArgumentParser, msg from jwlib.parse import parse_broadcasting from jwlib.download import download_all, disk_usage_info from jwlib.output import create_output @@ -29,24 +28,21 @@ parser.add_arguments(['--quiet', '--clean-symlinks', '--ntfs', '--download', + '--friendly', + '--download-subtitles', 'work_dir']) settings = parser.parse_args() -if not settings.mode and not settings.download: +if not (settings.mode or settings.download or settings.download_subtitles): msg('please use --mode or --download') exit(1) -# Convert download options to bools (what a mess) -settings.download_subtitles = settings.download in ('subtitles', 'friendly-subtitles') -settings.friendly_subtitle_filenames = settings.download == 'friendly-subtitles' -settings.download = settings.download == 'media' - # Some heads-up if settings.quiet < 1: if settings.download and settings.curl_path is not None and settings.rate_limit != '0': msg('note: download rate limit is active') - if not settings.safe_filenames and (settings.mode not in (None, 'stdout') or settings.friendly_subtitle_filenames): + if not settings.safe_filenames and (settings.mode not in (None, 'stdout') or settings.friendly_filenames): msg('note: NTFS/FAT compatibility is off') # Warning if disk space is already below limit diff --git a/jwlib/__init__.py b/jwlib/__init__.py index c121499..e69de29 100644 --- a/jwlib/__init__.py +++ b/jwlib/__init__.py @@ -1,5 +0,0 @@ -import sys as _sys - - -def msg(s): - print(s, file=_sys.stderr, flush=True) diff --git a/jwlib/arguments.py b/jwlib/arguments.py index 96d27ab..6adc633 100644 --- a/jwlib/arguments.py +++ b/jwlib/arguments.py @@ -1,9 +1,12 @@ +import sys import json import urllib.request import argparse import time -from . import msg + +def msg(s): + print(s, file=sys.stderr, flush=True) def action_factory(function): @@ -64,7 +67,7 @@ class Settings: # Download stuff download = False download_subtitles = False - friendly_subtitle_filenames = False + friendly_filenames = False curl_path = 'curl' rate_limit = '1M' checksums = False @@ -93,8 +96,13 @@ def __setattr__(self, key, value): class ArgumentParser(argparse.ArgumentParser): """Predefined arguments can be activated with add_arguments()""" - def add_predefined(self, *flags, **kwargs): - self.predefined_arguments[flags[0]] = dict(flags=flags, **kwargs) + def add_predefined(self, long_flag, short_flag=None, **kwargs): + # Note: put short flags first in help message + if short_flag: + flags = [short_flag, long_flag] + else: + flags = [long_flag] + self.predefined_arguments[long_flag] = dict(flags=flags, **kwargs) def add_arguments(self, flags: list): """Activate predefined arguments found in list""" @@ -121,12 +129,9 @@ def __init__(self, *args, argument_default=argparse.SUPPRESS, **kwargs): add_predefined('--mode', '-m', choices=['stdout', 'filesystem', 'm3u', 'm3ucompat', 'html'], help='output mode') - add_predefined('--lang', '-l', nargs='?', - action=action_factory(verify_language), + add_predefined('--lang', '-l', action=action_factory(verify_language), help='language code') - add_predefined('--languages', - nargs=0, - action=action_factory(print_language), + add_predefined('--languages', '-L', nargs=0, action=action_factory(print_language), help='display a list of valid language codes') add_predefined('--quality', '-Q', type=int, choices=[240, 360, 480, 720], @@ -154,7 +159,7 @@ def __init__(self, *args, argument_default=argparse.SUPPRESS, **kwargs): add_predefined('--since', metavar='YYYY-MM-DD', dest='min_date', action=action_factory(lambda x: time.mktime(time.strptime(x, '%Y-%m-%d'))), help='only index media newer than this date') - add_predefined('--limit-rate', dest='rate_limit', + add_predefined('--limit-rate', '-R', metavar='RATE', dest='rate_limit', help='maximum download rate, passed to curl (default: 1m = 1 megabyte/s, 0 = no limit)') add_predefined('--curl-path', metavar='PATH', help='path to the curl binary') @@ -162,11 +167,14 @@ def __init__(self, *args, argument_default=argparse.SUPPRESS, **kwargs): help='use urllib instead of external curl (compatibility)') add_predefined('--clean-symlinks', action='store_true', dest='clean_all_symlinks', help='remove all old symlinks (only valid with --mode=filesystem)') - add_predefined('--ntfs', action='store_true', dest='safe_filenames', + add_predefined('--ntfs', '-X', action='store_true', dest='safe_filenames', help='remove special characters from file names (NTFS/FAT compatibility)') - add_predefined('--download', '-d', nargs='?', const='media', - choices=['media', 'subtitles', 'friendly-subtitles'], - help='download media files or subtitles') + add_predefined('--download', '-d', action='store_true', + help='download media files') + add_predefined('--friendly', '-H', action='store_true', dest='friendly_filenames', + help='save downloads with human readable names') + add_predefined('--download-subtitles', action='store_true', + help='download VTT subtitle files') add_predefined('--forever', action='store_true', dest='stream_forever', help='re-run program when the last video finishes') add_predefined('work_dir', nargs='?', metavar='DIR', diff --git a/jwlib/download.py b/jwlib/download.py index 40167c4..e1afcd6 100644 --- a/jwlib/download.py +++ b/jwlib/download.py @@ -7,10 +7,8 @@ import urllib.parse from typing import List, Optional -from . import msg from .parse import Category, Media -from .arguments import Settings -from .output import format_filename +from .arguments import Settings, msg class MissingTimestampError(Exception): @@ -74,12 +72,8 @@ def download_all(s: Settings, data: List[Category]): def download_all_subtitles(s: Settings, media_list: List[Media], directory: str): - """Download VTT files from Media + """Download VTT files from Media""" - :param s: Global settings - :param media: a Media instance - :param directory: dir to save the files to - """ os.makedirs(directory, exist_ok=True) download_list = set() @@ -87,14 +81,10 @@ def download_all_subtitles(s: Settings, media_list: List[Media], directory: str) if not media.subtitle_url: continue - filename = os.path.basename(urllib.parse.urlparse(media.subtitle_url).path) - if s.friendly_subtitle_filenames: - filename = format_filename(media.name + os.path.splitext(filename)[1], safe=s.safe_filenames) - - file = os.path.join(directory, filename) + file = os.path.join(directory, media.subtitle_filename) # Note: --fix-broken will re-download all subtitle files... if s.overwrite_bad or not os.path.exists(file): - download_list.add((media.subtitle_url, file, filename)) + download_list.add((media.subtitle_url, file, media.subtitle_filename)) for num, data in enumerate(download_list): url, file, filename = data diff --git a/jwlib/output.py b/jwlib/output.py index 17de6d2..fd08efd 100644 --- a/jwlib/output.py +++ b/jwlib/output.py @@ -1,9 +1,8 @@ import os from typing import List -from . import msg from .parse import Category, Media -from .arguments import Settings +from .arguments import Settings, msg pj = os.path.join @@ -24,7 +23,7 @@ def create_output(s: Settings, data: List[Category], stdout_uniq=False): elif s.mode == 'm3ucompat': output_m3u(s, data, flat=True) elif s.mode == 'html': - output_m3u(s, data, writer=_write_to_html, file_ending='.html') + output_m3u(s, data, writer=_write_to_html, ext='.html') else: raise RuntimeError('invalid mode') @@ -48,39 +47,36 @@ def output_stdout(s: Settings, data: List[Category], uniq=False): print(*out, sep='\n') -def output_m3u(s: Settings, data: List[Category], writer=None, flat=False, file_ending='.m3u'): +def output_m3u(s: Settings, data: List[Category], writer=None, flat=False, ext='.m3u'): """Create a M3U playlist tree. :keyword writer: Function to write to files :keyword flat: If all playlist will be saved outside of subdir - :keyword file_ending: Well, duh + :keyword ext: Filename extension """ wd = s.work_dir sd = s.sub_dir - def fmt(x): - return format_filename(x, safe=s.safe_filenames) - if not writer: writer = _write_to_m3u for category in data: if flat: # Flat mode, all files in working dir - output_file = pj(wd, category.key + ' - ' + fmt(category.name) + file_ending) + output_file = pj(wd, category.key + ' - ' + category.safe_name + ext) source_prepend_dir = sd elif category.home: # For home/index/starting categories # The current file gets saved outside the subdir # Links point inside the subdir source_prepend_dir = sd - output_file = pj(wd, fmt(category.name) + file_ending) + output_file = pj(wd, category.safe_name + ext) else: # For all other categories # Things get saved inside the subdir # No need to prepend links with the subdir itself source_prepend_dir = '' - output_file = pj(wd, sd, category.key + file_ending) + output_file = pj(wd, sd, category.key + ext) is_start = True for item in category.contents: @@ -89,7 +85,7 @@ def fmt(x): # "flat" playlists does not link to other playlists continue name = item.name.upper() - source = pj('.', source_prepend_dir, item.key + file_ending) + source = pj('.', source_prepend_dir, item.key + ext) else: name = item.name if item.exists_in(pj(wd, sd)): @@ -111,9 +107,6 @@ def output_filesystem(s: Settings, data: List[Category]): wd = s.work_dir sd = s.sub_dir - def fmt(x): - return format_filename(x, safe=s.safe_filenames) - if s.quiet < 1: msg('creating directory structure') @@ -125,7 +118,7 @@ def fmt(x): # Index/starting/home categories: create link outside subdir if category.home: - link = pj(wd, fmt(category.name)) + link = pj(wd, category.safe_name) # Note: the source will be relative source = pj(sd, category.key) try: @@ -141,17 +134,16 @@ def fmt(x): source = pj('..', item.key) if s.include_keyname: - link = pj(output_dir, item.key + ' - ' + fmt(item.name)) + link = pj(output_dir, item.key + ' - ' + item.safe_name) else: - link = pj(output_dir, fmt(item.name)) + link = pj(output_dir, item.safe_name) else: if not item.exists_in(pj(wd, sd)): continue source = pj('..', item.filename) - ext = os.path.splitext(item.filename)[1] - link = pj(output_dir, fmt(item.name + ext)) + link = pj(output_dir, item.friendly_filename) try: os.symlink(source, link) @@ -159,20 +151,6 @@ def fmt(x): pass -def format_filename(string, safe=False): - """Remove unsafe characters from file names""" - - if safe: - # NTFS/FAT forbidden characters - string = string.replace('"', "'").replace(':', '.') - forbidden = '<>:"|?*/\0' - else: - # Unix forbidden characters - forbidden = '/\0' - - return ''.join(x for x in string if x not in forbidden) - - def clean_symlinks(s: Settings): """Clean out broken (or all) symlinks from work dir""" diff --git a/jwlib/parse.py b/jwlib/parse.py index 679cf0e..d5251da 100644 --- a/jwlib/parse.py +++ b/jwlib/parse.py @@ -1,4 +1,3 @@ -from sys import stderr import time import re import json @@ -8,8 +7,10 @@ from urllib.error import HTTPError from typing import List, Union -from . import msg -from .arguments import Settings +from .arguments import msg, Settings + +SAFE_FILENAMES = False +FRIENDLY_FILENAMES = False class Category: @@ -25,10 +26,13 @@ def __init__(self): def __repr__(self): return "Category('{}', {})".format(self.key, self.contents) + @property + def safe_name(self): + return format_filename(self.name) + class Media: """Object to put media info in.""" - url = None # type: str name = None # type: str md5 = None # type: str @@ -40,12 +44,46 @@ class Media: def __repr__(self): return "Media('{}')".format(self.filename) + def exists_in(self, directory): + return os.path.exists(os.path.join(directory, self.filename)) + + def _get_filename(self, url=''): + return format_filename(os.path.basename(urllib.parse.urlparse(url).path)) + + def _get_friendly_filename(self, url=''): + return format_filename((self.name or '') + os.path.splitext(self._get_filename(url))[1]) + @property def filename(self): - return os.path.basename(urllib.parse.urlparse(self.url).path) + if FRIENDLY_FILENAMES: + return self._get_friendly_filename(self.url) + else: + return self._get_filename(self.url) + + @property + def friendly_filename(self): + return self._get_friendly_filename(self.url) + + @property + def subtitle_filename(self): + if FRIENDLY_FILENAMES: + return self._get_friendly_filename(self.subtitle_url) + else: + return self._get_filename(self.subtitle_url) - def exists_in(self, directory): - return os.path.exists(os.path.join(directory, self.filename)) + +def format_filename(string): + """Remove unsafe characters from file names""" + + if SAFE_FILENAMES: + # NTFS/FAT forbidden characters + string = string.replace('"', "'").replace(':', '.') + forbidden = '<>:"|?*/\0' + else: + # Unix forbidden characters + forbidden = '/\0' + + return ''.join(x for x in string if x not in forbidden) # Whoops, copied this from the Kodi plug-in @@ -87,6 +125,11 @@ def parse_broadcasting(s: Settings): :param s: Global settings object """ + # TODO this is really ugly + global FRIENDLY_FILENAMES, SAFE_FILENAMES + FRIENDLY_FILENAMES = s.friendly_filenames + SAFE_FILENAMES = s.safe_filenames + result = [] # Make a copy of the list, because we'll append stuff here later