Extract cmd (#61)

* Add extract feature (#26), fix stdout/err blocking (#21)
* Other small fixes.

src/vorta/assets/UI/archivetab.ui

@@ -230,8 +230,38 @@
      </item>
     </layout>
    </item>
+   <item>
+    <spacer name="verticalSpacer">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeType">
+      <enum>QSizePolicy::Minimum</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>10</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
    <item>
     <layout class="QHBoxLayout" name="horizontalLayout_3">
+     <item>
+      <widget class="QPushButton" name="extractButton">
+       <property name="enabled">
+        <bool>true</bool>
+       </property>
+       <property name="text">
+        <string>Extract</string>
+       </property>
+       <property name="icon">
+        <iconset resource="../icons/collection.qrc">
+         <normaloff>:/icons/cloud-download.svg</normaloff>:/icons/cloud-download.svg</iconset>
+       </property>
+      </widget>
+     </item>
      <item>
       <widget class="QPushButton" name="mountButton">
        <property name="sizePolicy">
@@ -260,20 +290,6 @@
        </property>
       </widget>
      </item>
-     <item>
-      <widget class="QPushButton" name="extractButton">
-       <property name="enabled">
-        <bool>true</bool>
-       </property>
-       <property name="text">
-        <string>Extract</string>
-       </property>
-       <property name="icon">
-        <iconset resource="../icons/collection.qrc">
-         <normaloff>:/icons/cloud-download.svg</normaloff>:/icons/cloud-download.svg</iconset>
-       </property>
-      </widget>
-     </item>
      <item>
       <spacer name="horizontalSpacer">
        <property name="orientation">

src/vorta/assets/UI/extractdialog.ui

@@ -14,29 +14,28 @@
    <string>Dialog</string>
   </property>
   <layout class="QVBoxLayout" name="verticalLayout">
-   <item>
-    <widget class="QLabel" name="label">
-     <property name="text">
-      <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Archive: &lt;span style=&quot; font-weight:600;&quot;&gt;nyx2.local-2018-11-16T09:49:58 from November 16, 2018&lt;/span&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
-     </property>
-    </widget>
-   </item>
    <item>
     <layout class="QHBoxLayout" name="horizontalLayout_2">
      <property name="topMargin">
       <number>10</number>
      </property>
      <item>
-      <widget class="QPushButton" name="pushButton_4">
+      <widget class="QLabel" name="label_2">
        <property name="text">
-        <string>Select All</string>
+        <string>Archive:</string>
        </property>
       </widget>
      </item>
-     <item alignment="Qt::AlignLeft">
-      <widget class="QPushButton" name="pushButton_3">
+     <item>
+      <widget class="QLabel" name="archiveNameLabel">
+       <property name="font">
+        <font>
+         <weight>75</weight>
+         <bold>true</bold>
+        </font>
+       </property>
        <property name="text">
-        <string>Unselect All</string>
+        <string>nyx2.local-2018-11-16T09:49:58 from November 16, 2018</string>
        </property>
       </widget>
      </item>
@@ -53,23 +52,10 @@
        </property>
       </spacer>
      </item>
-     <item>
-      <widget class="QPushButton" name="pushButton_5">
-       <property name="text">
-        <string>Exclude Patterns</string>
-       </property>
-      </widget>
-     </item>
     </layout>
    </item>
    <item>
-    <widget class="QTreeWidget" name="fileTree">
-     <column>
-      <property name="text">
-       <string notr="true">1</string>
-      </property>
-     </column>
-    </widget>
+    <widget class="QTreeView" name="treeView"/>
    </item>
    <item>
     <layout class="QHBoxLayout" name="horizontalLayout">
@@ -90,7 +76,7 @@
       </spacer>
      </item>
      <item alignment="Qt::AlignRight">
-      <widget class="QPushButton" name="pushButton_2">
+      <widget class="QPushButton" name="cancelButton">
        <property name="text">
         <string>Cancel</string>
        </property>

src/vorta/borg/borg_thread.py

@@ -3,6 +3,9 @@
 import sys
 import shutil
 import signal
+import select
+import fcntl
+import time
 import logging
 from PyQt5 import QtCore
 from PyQt5.QtWidgets import QApplication
@@ -55,6 +58,7 @@ def __init__(self, cmd, params, parent=None):
 
         self.env = env
         self.cmd = cmd
+        self.cwd = params.get('cwd', None)
         self.params = params
         self.process = None
 
@@ -135,40 +139,74 @@ def run(self):
                                   profile=self.params.get('profile_name', None)
                                   )
         log_entry.save()
+        logger.info('Running command %s', ' '.join(self.cmd))
 
-        self.process = Popen(self.cmd, stdout=PIPE, stderr=PIPE, bufsize=1,
-                             universal_newlines=True, env=self.env, preexec_fn=os.setsid)
+        p = Popen(self.cmd, stdout=PIPE, stderr=PIPE, bufsize=1, universal_newlines=True,
+                  env=self.env, cwd=self.cwd, preexec_fn=os.setsid)
+        self.process = p
 
-        for line in iter(self.process.stderr.readline, ''):
+        # Prevent blocking. via https://stackoverflow.com/a/7730201/3983708
+
+        # Helper function to add the O_NONBLOCK flag to a file descriptor
+        def make_async(fd):
+            fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
+
+        # Helper function to read some data from a file descriptor, ignoring EAGAIN errors
+        def read_async(fd):
             try:
-                parsed = json.loads(line)
-                if parsed['type'] == 'log_message':
-                    self.log_event(f'{parsed["levelname"]}: {parsed["message"]}')
-                    level_int = getattr(logging, parsed["levelname"])
-                    logger.log(level_int, parsed["message"])
-                elif parsed['type'] == 'file_status':
-                    self.log_event(f'{parsed["path"]} ({parsed["status"]})')
-            except json.decoder.JSONDecodeError:
-                msg = line.strip()
-                self.log_event(msg)
-                logger.warning(msg)
-
-        self.process.wait()
-        stdout = self.process.stdout.read()
+                return fd.read()
+            except (IOError, TypeError):
+                return ''
+
+        make_async(p.stdout)
+        make_async(p.stderr)
+
+        stdout = []
+        while True:
+            # Wait for new output
+            select.select([p.stdout, p.stderr], [], [])
+
+            stdout.append(read_async(p.stdout))
+            stderr = read_async(p.stderr)
+            if stderr:
+                for line in stderr.split('\n'):
+                    try:
+                        parsed = json.loads(line)
+                        if parsed['type'] == 'log_message':
+                            self.log_event(f'{parsed["levelname"]}: {parsed["message"]}')
+                            level_int = getattr(logging, parsed["levelname"])
+                            logger.log(level_int, parsed["message"])
+                        elif parsed['type'] == 'file_status':
+                            self.log_event(f'{parsed["path"]} ({parsed["status"]})')
+                    except json.decoder.JSONDecodeError:
+                        msg = line.strip()
+                        self.log_event(msg)
+                        logger.warning(msg)
+
+            if p.poll() is not None:
+                stdout.append(read_async(p.stdout))
+                break
+
         result = {
             'params': self.params,
             'returncode': self.process.returncode,
-            'cmd': self.cmd
+            'cmd': self.cmd,
         }
+        stdout = ''.join(stdout)
+
         try:
             result['data'] = json.loads(stdout)
-        except:  # noqa
-            result['data'] = {}
+        except ValueError:
+            result['data'] = stdout
 
-        log_entry.returncode = self.process.returncode
+        log_entry.returncode = p.returncode
         log_entry.repo_url = self.params.get('repo_url', None)
         log_entry.save()
 
+        # Ensure async reading of mock stdout/stderr is finished.
+        if hasattr(sys, '_called_from_test'):
+            time.sleep(1)
+
         self.process_result(result)
         self.finished_event(result)
         mutex.unlock()
@@ -190,44 +228,3 @@ def started_event(self):
 
     def finished_event(self, result):
         self.result.emit(result)
-
-
-class BorgThreadChain(BorgThread):
-    """
-    Metaclass of `BorgThread` that can run multiple other BorgThread actions while providing the same
-    interface as a single action.
-    """
-
-    def __init__(self, cmds, input_values, parent=None):
-        """
-        Takes a list of tuples with `BorgThread` subclass and optional input parameters. Then all actions are executed
-        and a merged result object is returned to the caller. If there is any error, then current result is returned.
-
-        :param actions:
-        :return: dict(results)
-        """
-        self.parent = parent
-        self.threads = []
-        self.combined_result = {}
-
-        for cmd, input_value in zip(cmds, input_values):
-            if input_value is not None:
-                msg = cmd.prepare(input_value)
-            else:
-                msg = cmd.prepare()
-            if msg['ok']:
-                thread = cmd(msg['cmd'], msg, parent)
-                thread.updated.connect(self.updated.emit)  # All log entries are immediately sent to the parent.
-                thread.result.connect(self.partial_result)
-                self.threads.append(thread)
-        self.threads[0].start()
-
-    def partial_result(self, result):
-        if result['returncode'] == 0:
-            self.combined_result.update(result)
-            self.threads.pop(0)
-
-            if len(self.threads) > 0:
-                self.threads[0].start()
-            else:
-                self.result.emit(self.combined_result)

src/vorta/borg/create.py

@@ -67,9 +67,9 @@ def prepare(cls, profile):
                 (
                     WifiSettingModel.ssid == current_wifi
                 ) & (
-                    WifiSettingModel.allowed is False
+                    WifiSettingModel.allowed == False  # noqa
                 ) & (
-                    WifiSettingModel.profile == profile.id
+                    WifiSettingModel.profile == profile
                 )
             )
             if wifi_is_disallowed.count() > 0 and profile.repo.is_remote_repo():

src/vorta/borg/extract.py

@@ -0,0 +1,38 @@
+from .borg_thread import BorgThread
+
+
+class BorgExtractThread(BorgThread):
+
+    def log_event(self, msg):
+        self.app.backup_log_event.emit(msg)
+
+    def started_event(self):
+        self.app.backup_started_event.emit()
+        self.app.backup_log_event.emit('Downloading files from archive..')
+
+    def finished_event(self, result):
+        self.app.backup_finished_event.emit(result)
+        self.result.emit(result)
+        self.app.backup_log_event.emit('Restored files from archive.')
+
+    @classmethod
+    def prepare(cls, profile, archive_name, selected_files, destination_folder):
+        ret = super().prepare(profile)
+        if not ret['ok']:
+            return ret
+        else:
+            ret['ok'] = False  # Set back to false, so we can do our own checks here.
+
+        cmd = ['borg', 'extract', '--list', '--info', '--log-json']
+        cmd.append(f'{profile.repo.url}::{archive_name}')
+        for s in selected_files:
+            cmd.append(s)
+
+        ret['ok'] = True
+        ret['cmd'] = cmd
+        ret['cwd'] = destination_folder
+
+        return ret
+
+    def process_result(self, result):
+        pass

src/vorta/borg/list_archive.py

@@ -0,0 +1,32 @@
+from .borg_thread import BorgThread
+
+
+class BorgListArchiveThread(BorgThread):
+
+    def log_event(self, msg):
+        self.app.backup_log_event.emit(msg)
+
+    def started_event(self):
+        self.app.backup_started_event.emit()
+        self.app.backup_log_event.emit('Getting archive content..')
+
+    def finished_event(self, result):
+        self.app.backup_finished_event.emit(result)
+        self.app.backup_log_event.emit('Done getting archive content.')
+        self.result.emit(result)
+
+    @classmethod
+    def prepare(cls, profile):
+        ret = super().prepare(profile)
+        if not ret['ok']:
+            return ret
+        else:
+            ret['ok'] = False  # Set back to false, so we can do our own checks here.
+
+        cmd = ['borg', 'list', '--info', '--log-json', '--format', "{size:8d}{TAB}{mtime}{TAB}{path}{NL}"]
+        cmd.append(f'{profile.repo.url}')
+
+        ret['ok'] = True
+        ret['cmd'] = cmd
+
+        return ret

src/vorta/borg/list.py → src/vorta/borg/list_repo.py

@@ -3,7 +3,7 @@
 from vorta.models import ArchiveModel, RepoModel
 
 
-class BorgListThread(BorgThread):
+class BorgListRepoThread(BorgThread):
 
     def log_event(self, msg):
         self.app.backup_log_event.emit(msg)