Friendly download

- Media objects takes care of their file name, globally
- More short flags, tidy up help message
- Download flag is split up into three
- Move code out of __init__.py

jwb-index

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
-from jwlib import msg
-from jwlib.arguments import ArgumentParser
+from jwlib.arguments import ArgumentParser, msg
 from jwlib.parse import parse_broadcasting
 from jwlib.download import download_all, disk_usage_info
 from jwlib.output import create_output
@@ -29,24 +28,21 @@ parser.add_arguments(['--quiet',
                       '--clean-symlinks',
                       '--ntfs',
                       '--download',
+                      '--friendly',
+                      '--download-subtitles',
                       'work_dir'])
 
 settings = parser.parse_args()
 
-if not settings.mode and not settings.download:
+if not (settings.mode or settings.download or settings.download_subtitles):
     msg('please use --mode or --download')
     exit(1)
 
-# Convert download options to bools (what a mess)
-settings.download_subtitles = settings.download in ('subtitles', 'friendly-subtitles')
-settings.friendly_subtitle_filenames = settings.download == 'friendly-subtitles'
-settings.download = settings.download == 'media'
-
 # Some heads-up
 if settings.quiet < 1:
     if settings.download and settings.curl_path is not None and settings.rate_limit != '0':
         msg('note: download rate limit is active')
-    if not settings.safe_filenames and (settings.mode not in (None, 'stdout') or settings.friendly_subtitle_filenames):
+    if not settings.safe_filenames and (settings.mode not in (None, 'stdout') or settings.friendly_filenames):
         msg('note: NTFS/FAT compatibility is off')
 
 # Warning if disk space is already below limit

jwlib/__init__.py

@@ -1,5 +0,0 @@
-import sys as _sys
-
-
-def msg(s):
-    print(s, file=_sys.stderr, flush=True)

jwlib/arguments.py

@@ -1,9 +1,12 @@
+import sys
 import json
 import urllib.request
 import argparse
 import time
 
-from . import msg
+
+def msg(s):
+    print(s, file=sys.stderr, flush=True)
 
 
 def action_factory(function):
@@ -64,7 +67,7 @@ class Settings:
     # Download stuff
     download = False
     download_subtitles = False
-    friendly_subtitle_filenames = False
+    friendly_filenames = False
     curl_path = 'curl'
     rate_limit = '1M'
     checksums = False
@@ -93,8 +96,13 @@ def __setattr__(self, key, value):
 class ArgumentParser(argparse.ArgumentParser):
     """Predefined arguments can be activated with add_arguments()"""
 
-    def add_predefined(self, *flags, **kwargs):
-        self.predefined_arguments[flags[0]] = dict(flags=flags, **kwargs)
+    def add_predefined(self, long_flag, short_flag=None, **kwargs):
+        # Note: put short flags first in help message
+        if short_flag:
+            flags = [short_flag, long_flag]
+        else:
+            flags = [long_flag]
+        self.predefined_arguments[long_flag] = dict(flags=flags, **kwargs)
 
     def add_arguments(self, flags: list):
         """Activate predefined arguments found in list"""
@@ -121,12 +129,9 @@ def __init__(self, *args, argument_default=argparse.SUPPRESS, **kwargs):
         add_predefined('--mode', '-m',
                        choices=['stdout', 'filesystem', 'm3u', 'm3ucompat', 'html'],
                        help='output mode')
-        add_predefined('--lang', '-l', nargs='?',
-                       action=action_factory(verify_language),
+        add_predefined('--lang', '-l', action=action_factory(verify_language),
                        help='language code')
-        add_predefined('--languages',
-                       nargs=0,
-                       action=action_factory(print_language),
+        add_predefined('--languages', '-L', nargs=0, action=action_factory(print_language),
                        help='display a list of valid language codes')
         add_predefined('--quality', '-Q', type=int,
                        choices=[240, 360, 480, 720],
@@ -154,19 +159,22 @@ def __init__(self, *args, argument_default=argparse.SUPPRESS, **kwargs):
         add_predefined('--since', metavar='YYYY-MM-DD', dest='min_date',
                        action=action_factory(lambda x: time.mktime(time.strptime(x, '%Y-%m-%d'))),
                        help='only index media newer than this date')
-        add_predefined('--limit-rate', dest='rate_limit',
+        add_predefined('--limit-rate', '-R', metavar='RATE', dest='rate_limit',
                        help='maximum download rate, passed to curl (default: 1m = 1 megabyte/s, 0 = no limit)')
         add_predefined('--curl-path', metavar='PATH',
                        help='path to the curl binary')
         add_predefined('--no-curl', action='store_const', const=None, dest='curl_path',
                        help='use urllib instead of external curl (compatibility)')
         add_predefined('--clean-symlinks', action='store_true', dest='clean_all_symlinks',
                        help='remove all old symlinks (only valid with --mode=filesystem)')
-        add_predefined('--ntfs', action='store_true', dest='safe_filenames',
+        add_predefined('--ntfs', '-X', action='store_true', dest='safe_filenames',
                        help='remove special characters from file names (NTFS/FAT compatibility)')
-        add_predefined('--download', '-d', nargs='?', const='media',
-                       choices=['media', 'subtitles', 'friendly-subtitles'],
-                       help='download media files or subtitles')
+        add_predefined('--download', '-d', action='store_true',
+                       help='download media files')
+        add_predefined('--friendly', '-H', action='store_true', dest='friendly_filenames',
+                       help='save downloads with human readable names')
+        add_predefined('--download-subtitles', action='store_true',
+                       help='download VTT subtitle files')
         add_predefined('--forever', action='store_true', dest='stream_forever',
                        help='re-run program when the last video finishes')
         add_predefined('work_dir', nargs='?', metavar='DIR',

jwlib/download.py

@@ -7,10 +7,8 @@
 import urllib.parse
 from typing import List, Optional
 
-from . import msg
 from .parse import Category, Media
-from .arguments import Settings
-from .output import format_filename
+from .arguments import Settings, msg
 
 
 class MissingTimestampError(Exception):
@@ -74,27 +72,19 @@ def download_all(s: Settings, data: List[Category]):
 
 
 def download_all_subtitles(s: Settings, media_list: List[Media], directory: str):
-    """Download VTT files from Media
+    """Download VTT files from Media"""
 
-    :param s: Global settings
-    :param media: a Media instance
-    :param directory: dir to save the files to
-    """
     os.makedirs(directory, exist_ok=True)
 
     download_list = set()
     for media in media_list:
         if not media.subtitle_url:
             continue
 
-        filename = os.path.basename(urllib.parse.urlparse(media.subtitle_url).path)
-        if s.friendly_subtitle_filenames:
-            filename = format_filename(media.name + os.path.splitext(filename)[1], safe=s.safe_filenames)
-
-        file = os.path.join(directory, filename)
+        file = os.path.join(directory, media.subtitle_filename)
         # Note: --fix-broken will re-download all subtitle files...
         if s.overwrite_bad or not os.path.exists(file):
-            download_list.add((media.subtitle_url, file, filename))
+            download_list.add((media.subtitle_url, file, media.subtitle_filename))
 
     for num, data in enumerate(download_list):
         url, file, filename = data

jwlib/output.py

@@ -1,9 +1,8 @@
 import os
 from typing import List
 
-from . import msg
 from .parse import Category, Media
-from .arguments import Settings
+from .arguments import Settings, msg
 
 pj = os.path.join
 
@@ -24,7 +23,7 @@ def create_output(s: Settings, data: List[Category], stdout_uniq=False):
     elif s.mode == 'm3ucompat':
         output_m3u(s, data, flat=True)
     elif s.mode == 'html':
-        output_m3u(s, data, writer=_write_to_html, file_ending='.html')
+        output_m3u(s, data, writer=_write_to_html, ext='.html')
     else:
         raise RuntimeError('invalid mode')
 
@@ -48,39 +47,36 @@ def output_stdout(s: Settings, data: List[Category], uniq=False):
     print(*out, sep='\n')
 
 
-def output_m3u(s: Settings, data: List[Category], writer=None, flat=False, file_ending='.m3u'):
+def output_m3u(s: Settings, data: List[Category], writer=None, flat=False, ext='.m3u'):
     """Create a M3U playlist tree.
 
     :keyword writer: Function to write to files
     :keyword flat: If all playlist will be saved outside of subdir
-    :keyword file_ending: Well, duh
+    :keyword ext: Filename extension
     """
     wd = s.work_dir
     sd = s.sub_dir
 
-    def fmt(x):
-        return format_filename(x, safe=s.safe_filenames)
-
     if not writer:
         writer = _write_to_m3u
 
     for category in data:
         if flat:
             # Flat mode, all files in working dir
-            output_file = pj(wd, category.key + ' - ' + fmt(category.name) + file_ending)
+            output_file = pj(wd, category.key + ' - ' + category.safe_name + ext)
             source_prepend_dir = sd
         elif category.home:
             # For home/index/starting categories
             # The current file gets saved outside the subdir
             # Links point inside the subdir
             source_prepend_dir = sd
-            output_file = pj(wd, fmt(category.name) + file_ending)
+            output_file = pj(wd, category.safe_name + ext)
         else:
             # For all other categories
             # Things get saved inside the subdir
             # No need to prepend links with the subdir itself
             source_prepend_dir = ''
-            output_file = pj(wd, sd, category.key + file_ending)
+            output_file = pj(wd, sd, category.key + ext)
 
         is_start = True
         for item in category.contents:
@@ -89,7 +85,7 @@ def fmt(x):
                     # "flat" playlists does not link to other playlists
                     continue
                 name = item.name.upper()
-                source = pj('.', source_prepend_dir, item.key + file_ending)
+                source = pj('.', source_prepend_dir, item.key + ext)
             else:
                 name = item.name
                 if item.exists_in(pj(wd, sd)):
@@ -111,9 +107,6 @@ def output_filesystem(s: Settings, data: List[Category]):
     wd = s.work_dir
     sd = s.sub_dir
 
-    def fmt(x):
-        return format_filename(x, safe=s.safe_filenames)
-
     if s.quiet < 1:
         msg('creating directory structure')
 
@@ -125,7 +118,7 @@ def fmt(x):
 
         # Index/starting/home categories: create link outside subdir
         if category.home:
-            link = pj(wd, fmt(category.name))
+            link = pj(wd, category.safe_name)
             # Note: the source will be relative
             source = pj(sd, category.key)
             try:
@@ -141,38 +134,23 @@ def fmt(x):
                 source = pj('..', item.key)
 
                 if s.include_keyname:
-                    link = pj(output_dir, item.key + ' - ' + fmt(item.name))
+                    link = pj(output_dir, item.key + ' - ' + item.safe_name)
                 else:
-                    link = pj(output_dir, fmt(item.name))
+                    link = pj(output_dir, item.safe_name)
 
             else:
                 if not item.exists_in(pj(wd, sd)):
                     continue
 
                 source = pj('..', item.filename)
-                ext = os.path.splitext(item.filename)[1]
-                link = pj(output_dir, fmt(item.name + ext))
+                link = pj(output_dir, item.friendly_filename)
 
             try:
                 os.symlink(source, link)
             except FileExistsError:
                 pass
 
 
-def format_filename(string, safe=False):
-    """Remove unsafe characters from file names"""
-
-    if safe:
-        # NTFS/FAT forbidden characters
-        string = string.replace('"', "'").replace(':', '.')
-        forbidden = '<>:"|?*/\0'
-    else:
-        # Unix forbidden characters
-        forbidden = '/\0'
-
-    return ''.join(x for x in string if x not in forbidden)
-
-
 def clean_symlinks(s: Settings):
     """Clean out broken (or all) symlinks from work dir"""
 

jwlib/parse.py

@@ -1,4 +1,3 @@
-from sys import stderr
 import time
 import re
 import json
@@ -8,8 +7,10 @@
 from urllib.error import HTTPError
 from typing import List, Union
 
-from . import msg
-from .arguments import Settings
+from .arguments import msg, Settings
+
+SAFE_FILENAMES = False
+FRIENDLY_FILENAMES = False
 
 
 class Category:
@@ -25,10 +26,13 @@ def __init__(self):
     def __repr__(self):
         return "Category('{}', {})".format(self.key, self.contents)
 
+    @property
+    def safe_name(self):
+        return format_filename(self.name)
+
 
 class Media:
     """Object to put media info in."""
-
     url = None  # type: str
     name = None  # type: str
     md5 = None  # type: str
@@ -40,12 +44,46 @@ class Media:
     def __repr__(self):
         return "Media('{}')".format(self.filename)
 
+    def exists_in(self, directory):
+        return os.path.exists(os.path.join(directory, self.filename))
+
+    def _get_filename(self, url=''):
+        return format_filename(os.path.basename(urllib.parse.urlparse(url).path))
+
+    def _get_friendly_filename(self, url=''):
+        return format_filename((self.name or '') + os.path.splitext(self._get_filename(url))[1])
+
     @property
     def filename(self):
-        return os.path.basename(urllib.parse.urlparse(self.url).path)
+        if FRIENDLY_FILENAMES:
+            return self._get_friendly_filename(self.url)
+        else:
+            return self._get_filename(self.url)
+
+    @property
+    def friendly_filename(self):
+        return self._get_friendly_filename(self.url)
+
+    @property
+    def subtitle_filename(self):
+        if FRIENDLY_FILENAMES:
+            return self._get_friendly_filename(self.subtitle_url)
+        else:
+            return self._get_filename(self.subtitle_url)
 
-    def exists_in(self, directory):
-        return os.path.exists(os.path.join(directory, self.filename))
+
+def format_filename(string):
+    """Remove unsafe characters from file names"""
+
+    if SAFE_FILENAMES:
+        # NTFS/FAT forbidden characters
+        string = string.replace('"', "'").replace(':', '.')
+        forbidden = '<>:"|?*/\0'
+    else:
+        # Unix forbidden characters
+        forbidden = '/\0'
+
+    return ''.join(x for x in string if x not in forbidden)
 
 
 # Whoops, copied this from the Kodi plug-in
@@ -87,6 +125,11 @@ def parse_broadcasting(s: Settings):
 
     :param s: Global settings object
     """
+    # TODO this is really ugly
+    global FRIENDLY_FILENAMES, SAFE_FILENAMES
+    FRIENDLY_FILENAMES = s.friendly_filenames
+    SAFE_FILENAMES = s.safe_filenames
+
     result = []
 
     # Make a copy of the list, because we'll append stuff here later