diff --git a/.gitignore b/.gitignore index a358aa5..b7497b5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ make.bat config.py README.html CHANGELOG.html +webui.gz diff --git a/README.chs.md b/README.chs.md index 3193168..70b97aa 100755 --- a/README.chs.md +++ b/README.chs.md @@ -37,6 +37,7 @@ xeH - **download_ori** 是否下载原图，默认为否 - **jpn_title** 是否使用日语标题，如果关闭则使用英文或罗马字标题，默认为是 - **rename_ori** 将图片重命名为原始名称，如果关闭则使用序号，默认为否 + - **make_archive** 是否下载完成后生成zip压缩包，并删除下载目录，默认为否 高级参数: @@ -47,7 +48,6 @@ xeH - **rpc_port** RPC绑定的端口，默认为`None` - **rpc_secret** RPC密钥，默认为`None` (不开启RPC服务器) - **delete_task_files** 是否删除任务时同时删除下载的文件，默认为否 - - **make_archive** 是否下载完成后生成zip压缩包，并删除下载目录，默认为否 - **download_range** 设置下载的图片范围，参见[下载范围](#下载范围) - **scan_thread_cnt** 扫描线程数，默认为`1` - **download_thread_cnt** 下载线程数，默认为`5` @@ -64,9 +64,10 @@ xeH 用法: xeH [-u USERNAME] [-k KEY] [-c COOKIE] [-i] [--daemon] [-d DIR] [-o] [-j BOOL] [-r BOOL] [-p PROXY] [--proxy-image | --proxy-image-only] [--rpc-interface ADDR] [--rpc-port PORT] [--rpc-secret ...] - [--delete-task-files BOOL] [-a BOOL] [--download-range a-b,c-d,e] - [-t N] [--timeout N] [--low-speed-threshold N] [-f] - [-l /path/to/eh.log] [-v] [-h] [--version] + [--rpc-open-browser BOOL] [--delete-task-files BOOL] [-a BOOL] + [--download-range a-b,c-d,e] [-t N] [--timeout N] + [--low-speed-threshold N] [-f] [-l /path/to/eh.log] [-v] [-h] + [--version] [url [url ...]] 绅♂士下载器 @@ -96,6 +97,8 @@ xeH --rpc-interface ADDR 设置JSON-RPC监听IP (默认: localhost) --rpc-port PORT 设置JSON-RPC监听端口 (默认: None) --rpc-secret ... 设置JSON-RPC密钥 (默认: None) + --rpc-open-browser BOOL + RPC服务端启动后自动打开浏览器页面 (默认: True) --delete-task-files BOOL 删除任务时同时删除下载的文件 (默认: False) -a BOOL, --archive BOOL diff --git a/README.cht.md b/README.cht.md index d4e860f..67206ab 100644 --- a/README.cht.md +++ b/README.cht.md @@ -2,7 +2,7 @@ [![Build Status](https://travis-ci.org/fffonion/xeHentai.svg?branch=dev)](https://travis-ci.org/fffonion/xeHentai) -[English](README.md) [简体中文](README.chs.md) +[English](README.md) [繁體中文](README.cht.md) [xeHentai Web界面](https://github.com/fffonion/xeHentai-webui) @@ -37,6 +37,7 @@ xeH - **download_ori** 是否下載原圖，默認為否 - **jpn_title** 是否使用日語標題，如果關閉則使用英文或羅馬字標題，默認為是 - **rename_ori** 將圖片重命名為原始名稱，如果關閉則使用序號，默認為否 + - **make_archive** 是否下載完成後生成zip壓縮包，並刪除下載目錄，默認為否 高級參數: @@ -47,7 +48,6 @@ xeH - **rpc_port** RPC綁定的埠，默認為`None` - **rpc_secret** RPC密鑰，默認為`None` (不開啟RPC伺服器) - **delete_task_files** 是否刪除任務時同時刪除下載的文件，默認為否 - - **make_archive** 是否下載完成後生成zip壓縮包，並刪除下載目錄，默認為否 - **download_range** 設置下載的圖片範圍，參見[下載範圍](#下載範圍) - **scan_thread_cnt** 掃描線程數，默認為`1` - **download_thread_cnt** 下載線程數，默認為`5` @@ -64,9 +64,10 @@ xeH 用法: xeH [-u USERNAME] [-k KEY] [-c COOKIE] [-i] [--daemon] [-d DIR] [-o] [-j BOOL] [-r BOOL] [-p PROXY] [--proxy-image | --proxy-image-only] [--rpc-interface ADDR] [--rpc-port PORT] [--rpc-secret ...] - [--delete-task-files BOOL] [-a BOOL] [--download-range a-b,c-d,e] - [-t N] [--timeout N] [--low-speed-threshold N] [-f] - [-l /path/to/eh.log] [-v] [-h] [--version] + [--rpc-open-browser BOOL] [--delete-task-files BOOL] [-a BOOL] + [--download-range a-b,c-d,e] [-t N] [--timeout N] + [--low-speed-threshold N] [-f] [-l /path/to/eh.log] [-v] [-h] + [--version] [url [url ...]] 紳♂士下載器 @@ -96,6 +97,8 @@ xeH --rpc-interface ADDR 設置JSON-RPC監聽IP (默認: localhost) --rpc-port PORT 設置JSON-RPC監聽埠 (默認: None) --rpc-secret ... 設置JSON-RPC密鑰 (默認: None) + --rpc-open-browser BOOL + RPC服務端啟動後自動打開瀏覽器頁面 (默認: True) --delete-task-files BOOL 刪除任務時同時刪除下載的文件 (默認: False) -a BOOL, --archive BOOL diff --git a/README.md b/README.md index 653859d..a3c293e 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Configuration keys： - **download_ori** Set to download original images or not. Default to `False`. - **jpn_title** Set to select Japanese title or not. If set to `False`, English or Romaji title will be used. Default to `True`. - **rename_ori** Set to rename images to their orginal names. If set to `False`, image will be named in sequence numbers. Default to `False`. + - **make_archive** Set to make a ZIP archive after download and delete downloaded directory. Default to `False`. - **proxy** Proxy list. Refer to [Proxies](#proxies). - **proxy_image** Set to use proxy both on downloading images and scanning webpages. Default to `True`. @@ -44,8 +45,8 @@ Configuration keys： - **rpc_interface** RPC server binding IP. Refer to [JSON-RPC](#json-rpc). Default to `localhost`. - **rpc_port** RPC server binding port. Default to `none` (not serving). - **rpc_secret** RPC secret key. Default to `None`. + - **rpc_open_browser** automatically open browser after RPC server starts. Default to `True`. - **delete_task_files** Set to delete downloaded files when deleting a task. Default to `False`. - - **make_archive** Set to make a ZIP archive after download and delete downloaded directory. Default to `False`. - **download_range** Set image download range. Refer to [Download range](#download-range). Default to download all images. - **scan_thread_cnt** Thread count for scanning webpages. Default to `1`. - **download_thread_cnt** Thread count for downloading images. Default to `5`. @@ -62,9 +63,10 @@ Configuration keys： Usage: xeh [-u USERNAME] [-k KEY] [-c COOKIE] [-i] [--daemon] [-d DIR] [-o] [-j BOOL] [-r BOOL] [-p PROXY] [--proxy-image | --proxy-image-only] [--rpc-interface ADDR] [--rpc-port PORT] [--rpc-secret ...] - [--delete-task-files BOOL] [-a BOOL] [--download-range a-b,c-d,e] - [-t N] [--timeout N] [--low-speed-threshold N] [-f] - [-l /path/to/eh.log] [-v] [-h] [--version] + [--rpc-open-browser BOOL] [--delete-task-files BOOL] [-a BOOL] + [--download-range a-b,c-d,e] [-t N] [--timeout N] + [--low-speed-threshold N] [-f] [-l /path/to/eh.log] [-v] [-h] + [--version] [url [url ...]] xeHentai Downloader NG @@ -103,6 +105,9 @@ optional arguments: localhost) --rpc-port PORT bind jsonrpc server to this port (default: 8010) --rpc-secret ... jsonrpc secret string (default: None) + --rpc-open-browser BOOL + automatically open browser after RPC server starts + (default: True) --delete-task-files BOOL delete downloaded files when deleting a task (default: True) diff --git a/xeHentai/cli.py b/xeHentai/cli.py index fbf675b..060fc85 100644 --- a/xeHentai/cli.py +++ b/xeHentai/cli.py @@ -164,6 +164,8 @@ def parse_opt(): help = i18n.XEH_OPT_rpc_port) parser.add_argument('--rpc-secret', metavar = "...", default = _def['rpc_secret'], help = i18n.XEH_OPT_rpc_secret) + parser.add_argument('--rpc-open-browser', type = bool, metavar = "BOOL", default = _def['rpc_open_browser'], + help = i18n.XEH_OPT_rpc_open_browser) parser.add_argument('--delete-task-files', type = bool, metavar = "BOOL", default = _def['delete_task_files'], dest = 'delete_task_files', help = i18n.XEH_OPT_delete_task_files) parser.add_argument('-a', '--archive', type = bool, metavar = "BOOL", default = _def['make_archive'], diff --git a/xeHentai/config.py b/xeHentai/config.py index e5ed6fc..3f540d6 100644 --- a/xeHentai/config.py +++ b/xeHentai/config.py @@ -31,6 +31,8 @@ rpc_port = None # jsonrpc secret string rpc_secret = None +# auto open browser on rpc start +rpc_open_browser = True # make an archive (.zip) after download and delete directory make_archive = False diff --git a/xeHentai/const.py b/xeHentai/const.py index 2d0aee8..62131c5 100644 --- a/xeHentai/const.py +++ b/xeHentai/const.py @@ -32,6 +32,10 @@ DUMMY_FILENAME = "-dummy-" RENAME_TMPDIR = "-xeh-conflict-" +STATIC_CACHE_FILE = os.path.join(FILEPATH, "webui.gz") +# cache for 1 hour +STATIC_CACHE_TTL = 3600 +STATIC_CACHE_VERSION = 1 RE_INDEX = re.compile('.+/(\d+)/([^\/]+)/*') RE_GALLERY = re.compile('/([a-f0-9]{10})/[^\-]+\-(\d+)') diff --git a/xeHentai/core.py b/xeHentai/core.py index d18e236..4f0a392 100644 --- a/xeHentai/core.py +++ b/xeHentai/core.py @@ -82,6 +82,7 @@ def update_config(self, **cfg_dict): if not self.rpc and self.cfg['rpc_port'] and self.cfg['rpc_interface']: self.rpc = RPCServer(self, (self.cfg['rpc_interface'], int(self.cfg['rpc_port'])), secret = None if 'rpc_secret' not in self.cfg else self.cfg['rpc_secret'], + open_browser = False if 'rpc_open_browser' not in self.cfg else self.cfg['rpc_open_browser'], logger = self.logger) if not RE_LOCAL_ADDR.match(self.cfg['rpc_interface']) and \ not self.cfg['rpc_secret']: diff --git a/xeHentai/i18n/en_us.py b/xeHentai/i18n/en_us.py index de00a2e..a7d3678 100644 --- a/xeHentai/i18n/en_us.py +++ b/xeHentai/i18n/en_us.py @@ -58,6 +58,7 @@ XEH_OPT_rpc_interface = "bind jsonrpc server to this address (current: %(default)s)" XEH_OPT_rpc_port = "bind jsonrpc server to this port (current: %(default)s)" XEH_OPT_rpc_secret = "jsonrpc secret string (current: %(default)s)" +XEH_OPT_rpc_open_browser = "automatically open browser after RPC server starts (current: %(default)s)" XEH_OPT_a = "make an archive (.zip) after download and delete directory (current: %(default)s)" XEH_OPT_delete_task_files = "delete downloaded files when deleting a task (current: %(default)s)" XEH_OPT_j = "use Japanese title, use English/Romaji title if turned off (current: %(default)s)" @@ -119,6 +120,7 @@ RPC_STARTED = "RPC server listening on %s:%d" RPC_TOO_OPEN = "RPC server is listening on public interface (%s) but no rpc_secret defined, which is not safe" RPC_CANNOT_BIND = "RPC server can't listen on requested address: %s" +RPC_WEBUI_PATH = "WebUI is accessible at %s or https://xehentai.yooooo.us" SESSION_LOAD_EXCEPTION = "exception occurs when loading saved session: %s" SESSION_WRITE_EXCEPTION = "exception occurs when writing saved session: %s" diff --git a/xeHentai/i18n/zh_hans.py b/xeHentai/i18n/zh_hans.py index ec17446..7ca278c 100644 --- a/xeHentai/i18n/zh_hans.py +++ b/xeHentai/i18n/zh_hans.py @@ -56,6 +56,7 @@ XEH_OPT_rpc_interface = "设置JSON-RPC监听IP (当前: %(default)s)" XEH_OPT_rpc_port = "设置JSON-RPC监听端口 (当前: %(default)s)" XEH_OPT_rpc_secret = "设置JSON-RPC密钥 (当前: %(default)s)" +XEH_OPT_rpc_open_browser = "RPC服务端启动后自动打开浏览器页面 (当前: %(default)s)" XEH_OPT_a = "下载完成后生成zip压缩包并删除下载目录 (当前: %(default)s)" XEH_OPT_delete_task_files = "删除任务时同时删除下载的文件 (current: %(default)s)" XEH_OPT_j = "使用日语标题, 如果关闭则使用英文或罗马字标题 (当前: %(default)s)" @@ -118,6 +119,7 @@ RPC_STARTED = "RPC服务器监听在 %s:%d" RPC_TOO_OPEN = "RPC服务器监听在公网IP (%s)，为了安全起见应该设置rpc_secret" RPC_CANNOT_BIND = "RPC服务器无法启动：%s" +RPC_WEBUI_PATH = "WebUI 地址为 %s 或者 https://xehentai.yooooo.us" SESSION_LOAD_EXCEPTION = "读取存档时遇到错误: %s" SESSION_WRITE_EXCEPTION = "写入存档时遇到错误: %s" diff --git a/xeHentai/i18n/zh_hant.py b/xeHentai/i18n/zh_hant.py index f0911af..afe9a5a 100644 --- a/xeHentai/i18n/zh_hant.py +++ b/xeHentai/i18n/zh_hant.py @@ -56,6 +56,7 @@ XEH_OPT_rpc_interface = "設置JSON-RPC監聽IP (當前: %(default)s)" XEH_OPT_rpc_port = "設置JSON-RPC監聽埠 (當前: %(default)s)" XEH_OPT_rpc_secret = "設置JSON-RPC密鑰 (當前: %(default)s)" +XEH_OPT_rpc_open_browser = "RPC服務端啟動後自動打開瀏覽器頁面 (當前: %(default)s)" XEH_OPT_a = "下載完成後生成zip壓縮包並刪除下載目錄 (當前: %(default)s)" XEH_OPT_delete_task_files = "刪除任務時同時刪除下載的文件 (current: %(default)s)" XEH_OPT_j = "使用日語標題, 如果關閉則使用英文或羅馬字標題 (當前: %(default)s)" @@ -118,6 +119,7 @@ RPC_STARTED = "RPC伺服器監聽在 %s:%d" RPC_TOO_OPEN = "RPC伺服器監聽在公網IP (%s)，為了安全起見應該設置rpc_secret" RPC_CANNOT_BIND = "RPC伺服器無法啟動：%s" +RPC_WEBUI_PATH = "WebUI 地址為 %s 或者 https://xehentai.yooooo.us" SESSION_LOAD_EXCEPTION = "讀取存檔時遇到錯誤: %s" SESSION_WRITE_EXCEPTION = "寫入存檔時遇到錯誤: %s" diff --git a/xeHentai/rpc.py b/xeHentai/rpc.py index 17a86a8..dd04a91 100644 --- a/xeHentai/rpc.py +++ b/xeHentai/rpc.py @@ -10,6 +10,9 @@ import traceback from hashlib import md5 from threading import Thread +import zlib +import requests +import pickle from .const import * from .const import __version__ from .i18n import i18n @@ -26,12 +29,15 @@ from urlparse import urlparse cmdre = re.compile("([a-z])([A-Z])") -pathre = re.compile("/(?:jsonrpc|img|zip)") -imgpathre = re.compile("/img") -zippathre = re.compile("/zip") +pathre = re.compile("/(?:jsonrpc|img/|zip/|static/|ui/$)") +staticre = re.compile("/static/") +imgpathre = re.compile("/img/") +zippathre = re.compile("/zip/") + +version_str = "xeHentai/%s" % __version__ class RPCServer(Thread): - def __init__(self, xeH, bind_addr, secret = None, logger = None, exit_check = None): + def __init__(self, xeH, bind_addr, secret = None, open_browser = True, logger = None, exit_check = None): Thread.__init__(self, name = "rpc") Thread.setDaemon(self, True) self.xeH = xeH @@ -39,6 +45,7 @@ def __init__(self, xeH, bind_addr, secret = None, logger = None, exit_check = No self.secret = secret self.logger = logger self.server = None + self.open_browser = open_browser self._exit = exit_check if exit_check else lambda x:False def run(self): @@ -48,13 +55,22 @@ def run(self): self.logger.error(i18n.RPC_CANNOT_BIND % traceback.format_exc()) else: self.logger.info(i18n.RPC_STARTED % (self.bind_addr[0], self.bind_addr[1])) + url = "http://%s:%s/ui/#host=%s,port=%s,https=no" % ( + self.bind_addr[0], self.bind_addr[1], + self.bind_addr[0], self.bind_addr[1] + ) + if self.secret: + url = url + ",token=" + self.secret + if self.open_browser: + import webbrowser + webbrowser.open(url) + else: + self.logger.info(i18n.RPC_WEBUI_PATH % url) while not self._exit("rpc"): self.server.handle_request() -def is_file_obj(obj): - if PY3K: - return isinstance(obj, IOBase) - return isinstance(obj, file) +def is_readable_obj(obj): + return hasattr(obj, "read") def is_str_obj(obj): if PY3K: @@ -78,7 +94,7 @@ def gen_thumbnail(fh, args): return fh, False size = (int(args['w']) if 'w' in args else int(args['h']), int(args['h']) if 'h' in args else int(args['w'])) - if not is_file_obj(fh): + if not is_readable_obj(fh): fh = StringIO(fh) with Image.open(fh) as img: img.thumbnail(size) @@ -114,21 +130,42 @@ def f(self): func(self) return f +def load_cache(): + if os.path.exists(STATIC_CACHE_FILE): + try: + with open(STATIC_CACHE_FILE, "rb") as f: + r = zlib.decompress(f.read()) + r = pickle.loads(r) + if 'v' in r or r['v'] == STATIC_CACHE_VERSION: + return r + except: + pass + return { "v": STATIC_CACHE_VERSION } + +def save_cache(static_cache): + r = pickle.dumps(static_cache) + r = zlib.compress(r) + with open(STATIC_CACHE_FILE, "wb") as f: + f.write(r) + +static_cache = load_cache() class Handler(BaseHTTPRequestHandler): def __init__(self, xeH, secret, *args): self.secret = secret self.args = args self.xeH = xeHentaiRPCExtended(xeH, secret) + self.http = requests.Session() BaseHTTPRequestHandler.__init__(self, *args) def version_string(self): - return "xeHentai/%s" % __version__ + return version_str def serve_file(self, f): - _task = self.xeH._monitor.task - # needed to lock between archiver - _task._f_lock.acquire() + if hasattr(self.xeH, "_monitor"): + _task = self.xeH._monitor.task + # needed to lock between archiver + _task._f_lock.acquire() f.seek(0, os.SEEK_END) size = f.tell() self.xeH.logger.verbose("GET %s 200 %d %s" % (self.path, size, self.client_address[0])) @@ -140,7 +177,8 @@ def serve_file(self, f): if not buf: break self.wfile.write(buf) - _task._f_lock.release() + if hasattr(self.xeH, "_monitor"): + _task._f_lock.release() return size def do_OPTIONS(self): @@ -157,10 +195,11 @@ def do_GET(self): code = 200 rt = b'' mime = "text/html" + path = self.path while True: - if imgpathre.match(self.path): - args = dict(q.split("=") for q in urlparse(self.path).query.split("&") if q) - _ = urlparse(self.path).path.split("/") + if imgpathre.match(path): + args = dict(q.split("=") for q in urlparse(path).query.split("&") if q) + _ = urlparse(path).path.split("/") if len(_) < 5: code = 400 break @@ -191,9 +230,9 @@ def do_GET(self): rt, _error = gen_thumbnail(rt, args) if _error: self.xeH.logger.warning("RPC: PIL needed for generating thumbnail") - elif zippathre.match(self.path): + elif zippathre.match(path): # args = urlparse(_).query - _ = urlparse(self.path).path.split("/") + _ = urlparse(path).path.split("/") if len(_) < 5: code = 400 break @@ -211,6 +250,52 @@ def do_GET(self): code = 404 break rt = open(f, 'rb') + elif path == "/ui/" or staticre.match(path): + if path == "/ui/": + path = "/" + while True: + cache_rt = None + should_clear_cache = False + headers = { "User-Agent": version_str } + if path in static_cache: + cache_rt, mime, tm, lms = static_cache[path] + if PY3K and not isinstance(cache_rt, bytes): + cache_rt = bytes(cache_rt, 'ascii') + if time.time() - STATIC_CACHE_TTL < tm: + rt = StringIO(cache_rt) + break + should_clear_cache = True + headers['If-Modified-Since'] = lms + + req_start_tm = time.time() + r = None + try: + r = self.http.get("http://xehentai.yooooo.us%s?_=%d" %(path, time.time()), headers=headers) + except Exception as ex: + self.xeH.logger.warn("error pulling %s from remote server: %s" % (path, err)) + self.xeH.logger.verbose("%.2fs taken to pull %s from remote server %s bytes" % ( + time.time() - req_start_tm, path, r and len(r.content) or 0)) + if r and r.status_code == 200: + rt = StringIO(r.content) + mime = r.headers['Content-type'] + if should_clear_cache: + # clear all keys, since the js/css hash may change + static_cache.clear() + static_cache[path] = [r.content, mime, time.time(), r.headers['Last-Modified']] + save_cache(static_cache) + elif r and r.status_code == 304: + # so this is tricky: if we hit /ui/ first and it's not expired + # then all other assets should not expire + if path == "/": + for k in static_cache: + static_cache[k][2] = time.time() + rt = StringIO(cache_rt) + elif cache_rt: + self.xeH.logger.warn("serving stale cache %s" % (path)) + rt = StringIO(cache_rt) + else: + rt = jsonrpc_resp({"id":None}, error_code = ERR_RPC_INVALID_REQUEST) + break else: # fallback to rpc request rt = jsonrpc_resp({"id":None}, error_code = ERR_RPC_INVALID_REQUEST) @@ -221,7 +306,7 @@ def do_GET(self): self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Content-Type", mime) - if is_file_obj(rt): + if is_readable_obj(rt): size = self.serve_file(rt) rt.close() else: @@ -283,7 +368,10 @@ def do_POST(self): break self.xeH.logger.verbose("RPC from: %s, cmd: %s, params: %s" % (self.client_address[0], cmd, params)) try: - cmd_rt = getattr(self.xeH, cmd_r)(*params[-2], **params[-1]) + # pop out token if extra token is found + if len(params[0]) > 0 and 'token:' in params[0][0]: + del params[0][0] + cmd_rt = getattr(self.xeH, cmd_r)(*params[0], **params[1]) except (ValueError, TypeError) as ex: self.xeH.logger.verbose("RPC exec error:\n%s" % traceback.format_exc()) code = 500