many updates

.gitignore

@@ -1,12 +1,12 @@
 *.py[cod]
 
-.atomignore
 *.pyc
 *.json
 *.sh
 *.log
 release
 desktop.ini
 verinfo.txt
-config.py
+.atomignore
 make.bat
+config.py

xeHentai/cli.py

@@ -29,9 +29,9 @@ def start():
         if os.name == "posix":
             pid = os.fork()
             if pid == 0:
-                sys.stdin.close()
-                sys.stdout = open("/dev/null", "w")
-                sys.stderr = open("/dev/null", "w")
+                #sys.stdin.close()
+                #sys.stdout = open("/dev/null", "w")
+                #sys.stderr = open("/dev/null", "w")
                 return main(xeH, opt)
         elif os.name == "nt":
             import multiprocessing
@@ -63,20 +63,21 @@ def main(xeH, opt):
     try:
         if opt.urls:
             for u in opt.urls:
-                xeH.add_task(u)
+                xeH.add_task(u.strip())
             # finished this task and exit xeHentai
             Thread(target = lambda:(time.sleep(0.618), setattr(xeH, "_exit", XEH_STATE_SOFT_EXIT))).start()
         Thread(target = xeH._task_loop, name = "main" ).start()
         while xeH._exit < XEH_STATE_CLEAN:
             time.sleep(1)
     except KeyboardInterrupt:
         log.info(i18n.XEH_CLEANUP)
+        xeH._term_threads()
     except Exception as ex:
         log.error(i18n.XEH_CRITICAL_ERROR % traceback.format_exc())
+        xeH._term_threads()
     else:
         sys.exit(0) # this is mandatory for single task auto exit
     try:
-        xeH._term_threads()
         # we should call cleanup ourself because we break out of task_loop
         xeH._cleanup()
     except KeyboardInterrupt:
@@ -148,4 +149,5 @@ def _readline(s):
     download_ori = _readline(i18n.PS_DOWNLOAD_ORI) == "y"
     proxy = _readline(i18n.PS_PROXY).strip()
     proxy = [proxy] if proxy else None
-    return {'urls': url, 'proxy': proxy, 'download_ori': download_ori}
+    _dir = _readline(i18n.PS_DOWNLOAD_DIR % os.path.abspath(xeH.cfg['dir'])) or xeH.cfg['dir']
+    return {'urls': url, 'proxy': proxy, 'download_ori': download_ori, 'dir': _dir}

xeHentai/config.py

@@ -0,0 +1,17 @@
+# coding:utf-8
+# DO NOT EDIT THIS FILE
+# make a copy to your working directory
+# and edit that file
+
+dir = "."
+download_ori = False
+download_thread_cnt = 5
+scan_thread_cnt = 1
+proxy = None
+log_path = "eh.log"
+log_verbose = 2
+daemon = False
+fast_scan = False
+rpc_interface = 'localhost'
+rpc_port = None
+rpc_secret = None

xeHentai/const.py

@@ -63,5 +63,5 @@
 ERR_RPC_PARSE_ERROR = -32700
 ERR_RPC_INVALID_REQUEST = -32600
 ERR_RPC_METHOD_NOT_FOUND = -32601
-ERR_RPC_INVALID_PARAMS = -32601
+ERR_RPC_INVALID_PARAMS = -32602
 ERR_RPC_EXEC_ERROR = -32603

xeHentai/core.py

@@ -53,8 +53,8 @@ def __init__(self):
 
     def update_config(self, cfg_dict):
         self.cfg.update({k:v for k, v in cfg_dict.iteritems() if k in cfg_dict})
-        self.logger.verbose("cfg %s" % self.cfg)
         self.logger.set_level(logger.Logger.WARNING - self.cfg['log_verbose'])
+        self.logger.verbose("cfg %s" % self.cfg)
         if cfg_dict['proxy']:
             if not self.proxy: # else we keep it None
                 self.proxy = proxy.Pool()

xeHentai/i18n/en_us.py

@@ -33,7 +33,7 @@
 XEH_OPT_c = "cookie string, will be overriden if given -u and -k"
 XEH_OPT_o = "download original images (current: %(default)s)"
 XEH_OPT_t = "download threads count (current: %(default)d)"
-XEH_OPT_f = "guess page url from .hathdl file, not working everytime (current: %(default)s)"
+XEH_OPT_f = "fast scan, guess page url from .hathdl file, not working everytime (current: %(default)s)"
 XEH_OPT_l = "define log path (current: %(default)s)"
 XEH_OPT_p = "set download proxies, currenlty supported: socks5/4a, http(s) (current: %(default)s)"
 XEH_OPT_d = "set download directory (current: %(default)s)"
@@ -47,9 +47,10 @@
 PS_LOGIN = "login to exhentai (y/n)? > "
 PS_USERNAME = "Username > "
 PS_PASSWD = "Password > "
-PS_URL = "URL > "
+PS_URL = "URL (seperate with ,)> "
 PS_PROXY = "Proxy (optional) > "
 PS_DOWNLOAD_ORI = "Download original (y/n)? > "
+PS_DOWNLOAD_DIR = "Download to (current: %s), press enter or enter new > "
 
 PROXY_CANDIDATE_CNT = "proxy pool has %d candidates"
 

xeHentai/i18n/zh_cn.py

@@ -3,9 +3,27 @@
 from ..const import *
 
 err_msg = {
-
+    ERR_URL_NOT_RECOGNIZED: "url not recognized",
+    ERR_CANT_DOWNLOAD_EXH: "can't download exhentai.org without login",
+    ERR_ONLY_VISIBLE_EXH: "this gallery is only visible in exhentai.org",
+    ERR_MALFORMED_HATHDL: "malformed .hathdl, can't parse",
+    ERR_GALLERY_REMOVED: "this gallery has been removed, may be visible in exhentai",
+    ERR_NO_PAGEURL_FOUND: "no page url found, change of site structure?",
+    ERR_TASK_NOT_FOUND: "no such task guid",
+    ERR_TASK_LEVEL_UNDEF: "task filter level unknown",
+    ERR_DELETE_RUNNING_TASK: "can't delete a running task",
+    ERR_TASK_CANNOT_PAUSE: "this task can't be paused",
+    ERR_TASK_CANNOT_RESUME: "this task can't be resumed",
+    ERR_RPC_PARSE_ERROR: "Parse error.",
+    ERR_RPC_INVALID_REQUEST: "Invalid request.",
+    ERR_RPC_METHOD_NOT_FOUND: "Method not found.",
+    ERR_RPC_INVALID_PARAMS: "Invalid method parameter(s).",
+    ERR_RPC_EXEC_ERROR: "",
+    ERR_SAVE_SESSION_FAILED: "",
 }
 
+ERR_NOMSG = "undefined error message with code %d"
+
 XEH_OPT_DESC = "绅♂士下载器"
 XEH_OPT_EPILOG = "如果参数未指定，则使用config.py中的默认值"
 XEH_OPT_URLS = "下载页的网址"
@@ -14,7 +32,7 @@
 XEH_OPT_c = "Cookie字符串，如果指定了用户名和密码，此项会被忽略"
 XEH_OPT_o = "是否下载原始图片（如果存在） (当前: %(default)s)"
 XEH_OPT_t = "下载线程数 (当前: %(default)d)"
-XEH_OPT_f = "从hathdl猜测页面链接，可以提高抓取速度，但有时会抽风 (当前: %(default)s)"
+XEH_OPT_f = "快速扫描，从hathdl猜测页面链接，但有时会抽风 (当前: %(default)s)"
 XEH_OPT_l = "保存日志的路径 (当前: %(default)s)"
 XEH_OPT_p = "设置代理, 当前支持的类型: socks5/4a, http(s) (当前: %(default)s)"
 XEH_OPT_d = "设置下载目录 (当前: %(default)s)"
@@ -32,5 +50,38 @@
 PS_URL = "输入地址（使用,分割下载多个）> "
 PS_PROXY = "输入代理地址 (可选) > "
 PS_DOWNLOAD_ORI = "是否下载原图（默认否） (y/n)? > "
+PS_DOWNLOAD_DIR = "下载目录 (当前: %s)\n回车确认或输入新路径 > "
+
+PROXY_CANDIDATE_CNT = "代理池中有%d个代理"
+
+TASK_PUT_INTO_WAIT = "任务 #%s 已存在, 加入等待队列"
+TASK_ERROR = "任务 #%s 发生错误: %s"
+TASK_MIGRATE_EXH = "任务 #%s 使用里站地址重新下载"
+TASK_TITLE = "任务 #%s 标题 %s"
+TASK_WILL_DOWNLOAD_CNT = "任务 #%s 将下载%d个文件，共%d个 "
+TASK_START = "任务 #%s 开始"
+TASK_FINISHED = "任务 #%s 完成"
+TASK_START_PAGE_RESCAN = "任务 #%s 图片被缩放，进行完整扫描"
+TASK_FAST_SCAN = "任务 #%s 使用快速扫描"
 
 XEH_STARTED = "xeHentai %s 已启动"
+XEH_LOOP_FINISHED = "程序循环已完成"
+XEH_LOGIN_EXHENTAI = "登录绅士"
+XEH_LOGIN_OK = "已成为绅士"
+XEH_LOGIN_FAILED = "无法登录绅士"
+XEH_LOAD_TASKS_CNT = "从存档中读取了%d个任务"
+XEH_DAEMON_START = "后台进程已启动，PID为%d"
+XEH_RPC_STARTED = "rpc 服务器监听在 %s:%d"
+XEH_PLATFORM_NO_DAEMON = "后台模式不支持您的系统: %s"
+XEH_CLEANUP = "擦干净..."
+XEH_CRITICAL_ERROR = "xeHentai 抽风啦:\n%s"
+
+SESSION_LOAD_EXCEPTION = "读取存档时遇到错误: %s"
+SESSION_WRITE_EXCEPTION = "写入存档时遇到错误: %s"
+
+THREAD = "绅士"
+THREAD_UNCAUGHT_EXCEPTION = "绅士-%s 未捕获的异常\n%s"
+THREAD_MAY_BECOME_ZOMBIE = "绅士-%s 可能变成了丧尸"
+THREAD_SWEEP_OUT = "绅士-%s 挂了, 不再理它"
+
+QUEUE = "队列"

xeHentai/rpc.py

@@ -10,9 +10,11 @@
 from SocketServer import ThreadingMixIn
 from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
 from .const import *
+from .const import __version__
 from .i18n import i18n
 
 cmdre = re.compile("([a-z])([A-Z])")
+pathre = re.compile("/jsonrpc")
 
 class RPCServer(Thread):
     def __init__(self, xeH, bind_addr, secret = None, logger = None, exit_check = None):
@@ -47,7 +49,7 @@ def jsonrpc_resp(request, ret = None, error_code = None, error_msg = None):
 
 def path_filter(func):
     def f(self):
-        if self.path != "/jsonrpc":
+        if not pathre.match(self.path):
             self.send_response(404)
             self.end_headers()
             self.wfile.write('\n')
@@ -60,8 +62,18 @@ class Handler(BaseHTTPRequestHandler):
     def __init__(self, xeH, *args):
         self.xeH = xeH
         self.args = args
+        self.server_version = "xeHentai/%s" % __version__
         BaseHTTPRequestHandler.__init__(self, *args)
 
+    def do_OPTIONS(self):
+        self.send_response(200)
+        self.send_header("Access-Control-Allow-Origin", "*")
+        self.send_header("Access-Control-Allow-Headers", "Content-Type")
+        self.send_header("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
+        self.send_header("Access-Control-Max-Age", "1728000")
+        self.end_headers()
+        self.wfile.write('\n')
+
     @path_filter
     def do_GET(self):
         self.send_response(400)
@@ -74,29 +86,35 @@ def do_GET(self):
     def do_POST(self):
         d = self.rfile.read(int(self.headers.getheader('Content-Length')))
         rt = None
+        code = 200
         while True:
             try:
                 j = json.loads(d)
                 assert('method' in j and j['method'] != None and 'id' in j)
             except ValueError:
-                rt = jsonrpc_resp({"id":None}, error_code = ERR_RPC_PARSE_ERROR)
+                code = 400
+                rte = jsonrpc_resp({"id":None}, error_code = ERR_RPC_PARSE_ERROR)
                 break
             except AssertionError:
+                code = 400
                 rt = jsonrpc_resp({"id":None}, error_code = ERR_RPC_INVALID_REQUEST)
                 break
             cmd = re.findall("xeH\.(.+)", j['method'])
             if not cmd:
+                code = 404
                 rt = jsonrpc_resp({"id":j['id']}, error_code = ERR_RPC_METHOD_NOT_FOUND)
                 break
             # let's make fooBar to foo_bar
             cmd_r = cmdre.sub(lambda m: "%s_%s" % (m.group(1), m.group(2).lower()), cmd[0])
             if not hasattr(self.xeH, cmd_r) or cmd_r.startswith("_"):
+                code = 404
                 rt = jsonrpc_resp({"id":j['id']}, error_code = ERR_RPC_METHOD_NOT_FOUND)
                 break
             try:
                 params = ([], {}) if 'params' not in j else j['params']
                 cmd_rt = getattr(self.xeH, cmd_r)(*params[0], **params[1])
             except ValueError as ex:
+                code = 500
                 rt = jsonrpc_resp({"id":j['id']}, error_code = ERR_RPC_EXEC_ERROR,
                 error_msg = str(ex))
                 break
@@ -105,6 +123,8 @@ def do_POST(self):
             else:
                 rt = jsonrpc_resp({"id":j['id']}, ret = cmd_rt[1])
             break
+        self.send_response(code)
+        self.end_headers()
         self.wfile.write(rt)
         self.wfile.write('\n')
         return

xeHentai/task.py

@@ -89,7 +89,6 @@ def get_reload_url(self, imgurl):
         return self.reload_map[imgurl][0]
 
     def scan_downloaded(self, scaled = True):
-        print(self.config)
         fpath = os.path.join(self.config['dir'], util.legalpath(self.meta['title']))
         if not os.path.exists(fpath):
             return

xeHentai/util/logger.py

@@ -91,7 +91,7 @@ def set_logfile(self, fpath):
 
     def set_level(self, level):
         f = ('verbose', 'debug', 'info')
-        lv = min(max(level, 0), 2)
+        lv = min(max(level, 0), 3)
         for p in range(lv):
             setattr(self, f[p], self.dummy)
 

xeHentai/worker.py

@@ -174,7 +174,7 @@ def run(self):
             intv += 1
             self._check_vote()
             for k in self.thread_last_seen.keys():
-                if time.time() - self.thread_last_seen[k] > 15:
+                if time.time() - self.thread_last_seen[k] > 30:
                     if k in self.thread_ref and self.thread_ref[k].is_alive():
                         self.logger.warning(i18n.THREAD_MAY_BECOME_ZOMBIE % k)
                         self.thread_zombie.add(k)