Adds Keybase engine

Allows syncing files to Keybase.

Folders mounted with Keybase utilize a special file system (kbfs) which,
while doing some pretty cool stuff, either doesn't support some common
operations or requires them to be done via the Keybase CLI. The two
which pertain to Mackup are copying folders recursively (which should be
done with `keybase fs cp`) and stripping ACL's (which is not at all
supported). I tried to document these oddities pretty well, but let me
know if this could be made more clear.

Closes lra#1048

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## WIP
 
 - Add support for Boxer (via @icopp)
+- Add Keybase engine (via @jutonz)
 
 ## Mackup 0.8.16
 

README.md

@@ -145,6 +145,7 @@ in it stay put, so that any other computer also running Mackup is unaffected.
 - [Copy](https://www.copy.com/)
 - [iCloud](http://www.apple.com/icloud/)
 - [Box](https://www.box.com)
+- [Keybase](https://www.keybase.io)
 - Anything able to sync a folder (e.g. [Git](http://git-scm.com/))
 
 See the [README](doc/README.md) file in the doc directory for more info.

doc/README.md

@@ -63,6 +63,16 @@ where your Copy folder is and store your configuration files in it.
 engine = copy
 ```
 
+### Keybase
+
+```ini
+[storage]
+engine = keybase
+username = yourusername
+```
+
+This config will sync files to `/keybase/private/yourusername/Mackup`
+
 ### File System
 
 If you want to specify another directory, you can use the `file_system` engine

mackup/config.py

@@ -11,14 +11,16 @@
                         ENGINE_COPY,
                         ENGINE_ICLOUD,
                         ENGINE_BOX,
+                        ENGINE_KEYBASE,
                         ENGINE_FS)
 
 from .utils import (error,
                     get_dropbox_folder_location,
                     get_copy_folder_location,
                     get_google_drive_folder_location,
                     get_icloud_folder_location,
-                    get_box_folder_location)
+                    get_box_folder_location,
+                    get_keybase_folder_location)
 try:
     import configparser
 except ImportError:
@@ -65,8 +67,8 @@ def engine(self):
         """
         The engine used by the storage.
 
-        ENGINE_DROPBOX, ENGINE_GDRIVE, ENGINE_COPY, ENGINE_ICLOUD, ENGINE_BOX
-        or ENGINE_FS.
+        ENGINE_DROPBOX, ENGINE_GDRIVE, ENGINE_COPY, ENGINE_ICLOUD, ENGINE_BOX,
+        ENGINE_KEYBASE, or ENGINE_FS.
 
         Returns:
             str
@@ -188,6 +190,7 @@ def _parse_engine(self):
                           ENGINE_COPY,
                           ENGINE_ICLOUD,
                           ENGINE_BOX,
+                          ENGINE_KEYBASE,
                           ENGINE_FS]:
             raise ConfigError('Unknown storage engine: {}'.format(engine))
 
@@ -210,6 +213,12 @@ def _parse_path(self):
             path = get_icloud_folder_location()
         elif self.engine == ENGINE_BOX:
             path = get_box_folder_location()
+        elif self.engine == ENGINE_KEYBASE:
+            if not self._parser.has_option('storage', 'username'):
+                raise ConfigError("Also specify a 'username' key when using"
+                                  " 'engine = keybase')")
+            username = self._parser.get('storage', 'username')
+            path = get_keybase_folder_location(username)
         elif self.engine == ENGINE_FS:
             if self._parser.has_option('storage', 'path'):
                 cfg_path = self._parser.get('storage', 'path')

mackup/constants.py

@@ -28,3 +28,4 @@
 ENGINE_FS = 'file_system'
 ENGINE_GDRIVE = 'google_drive'
 ENGINE_ICLOUD = 'icloud'
+ENGINE_KEYBASE = 'keybase'

mackup/main.py

@@ -72,6 +72,9 @@ def printAppHeader(app_name):
     if args['--force']:
         utils.FORCE_YES = True
 
+    # Allow utils to make decisions based on config
+    utils.CONFIG = mckp._config
+
     dry_run = args['--dry-run']
 
     verbose = args['--verbose']

mackup/utils.py

@@ -15,6 +15,8 @@
 # If True, the user wants to say "yes" to everything.
 FORCE_YES = False
 
+CONFIG = None  # Overridden once config is read
+
 
 def confirm(question):
     """
@@ -56,7 +58,8 @@ def delete(filepath):
         filepath (str): Absolute full path to a file. e.g. /path/to/file
     """
     # Some files have ACLs, let's remove them recursively
-    remove_acl(filepath)
+    if can_remove_acl_with_current_engine():
+        remove_acl(filepath)
 
     # Some files have immutable attributes, let's remove them recursively
     remove_immutable_attribute(filepath)
@@ -97,12 +100,11 @@ def copy(src, dst):
 
     # We need to copy a single file
     if os.path.isfile(src):
-        # Copy the src file to dst
         shutil.copy(src, dst)
 
     # We need to copy a whole folder
     elif os.path.isdir(src):
-        shutil.copytree(src, dst)
+        _copy_recursive(src, dst)
 
     # What the heck is this ?
     else:
@@ -112,6 +114,22 @@ def copy(src, dst):
     chmod(dst)
 
 
+def _copy_recursive(src, dst):
+    """
+    Helper for copy, above.
+
+    Keybase uses a special file system to mount synced directories securely.
+    This file system does not yet allow for directories to be copied into it
+    recursively using `cp`, so in those situations we defer to the Keybase CLI
+    to do the heavy lifting. Note that this only necessary when copying
+    directories into the store--it's okay to use `cp` when copying files out.
+    """
+    if current_engine() == 'keybase' and '/keybase' in dst:
+        subprocess.check_output(['keybase', 'fs', 'cp', src, dst])
+    else:
+        shutil.copytree(src, dst)
+
+
 def link(target, link_to):
     """
     Create a link to a target file or a folder.
@@ -269,6 +287,16 @@ def get_box_folder_location():
     return box_home
 
 
+def get_keybase_folder_location(username):
+    """
+    Try to locate the Keybase folder.
+
+    Returns:
+        (str) Full path to the current Keybase folder.
+    """
+    return '/keybase/private/' + username
+
+
 def get_copy_folder_location():
     """
     Try to locate the Copy folder.
@@ -412,3 +440,15 @@ def can_file_be_synced_on_current_platform(path):
             can_be_synced = False
 
     return can_be_synced
+
+
+def can_remove_acl_with_current_engine():
+    # Keybase doesn't support stripping ACL's (chmod prints an ugly error)
+    return CONFIG and CONFIG.engine != 'keybase'
+
+
+def current_engine():
+    if not CONFIG:
+        return None
+    else:
+        return CONFIG.engine

requirements.txt

@@ -3,4 +3,5 @@
 nose
 flake8
 codecov
-docopt
+docopt
+mock

tests/utils_test.py

@@ -2,8 +2,10 @@
 import tempfile
 import unittest
 import stat
+import subprocess
+import random
 
-# from unittest.mock import patch
+from mock import Mock, patch
 
 from mackup import utils
 
@@ -88,6 +90,27 @@ def test_delete_folder_recursively(self):
         assert not os.path.exists(subfolder_path)
         assert not os.path.exists(subfilepath)
 
+    @patch.object(utils, 'can_remove_acl_with_current_engine', Mock(return_value=False))
+    @patch.object(utils, 'remove_acl', Mock())
+    def test_delete_does_not_remove_acl(self):
+        """
+        utils.delete does not strip ACL's if the engine does not support it
+        """
+        # Create a tmp file
+        tfile = tempfile.NamedTemporaryFile(delete=False)
+        tfpath = tfile.name
+        tfile.close()
+
+        # Make sure the created file exists
+        assert os.path.isfile(tfpath)
+
+        # Check if mackup can really delete it
+        utils.delete(tfpath)
+        assert not os.path.exists(tfpath)
+
+        # Make sure we didn't try to strip acl's
+        utils.remove_acl.assert_not_called()
+
     def test_copy_file(self):
         # Create a tmp file
         tfile = tempfile.NamedTemporaryFile(delete=False)
@@ -217,6 +240,50 @@ def test_copy_dir(self):
         utils.delete(srcpath)
         utils.delete(dstpath)
 
+    @patch('subprocess.check_output', Mock(return_value=0))
+    @patch.object(utils, 'chmod', Mock(return_value=0))
+    @patch.object(utils, 'CONFIG', Mock())
+    def test_copy_dir_keybase(self):
+        """Use `keybase fs cp` when copying directories into the store"""
+        # Set engine to Keybase
+        utils.CONFIG.engine = 'keybase'
+
+        # Create a temp folder
+        srcpath = tempfile.mkdtemp()
+
+        # Create a temp file
+        tfile = tempfile.NamedTemporaryFile(delete=False, dir=srcpath)
+        srcfile = tfile.name
+        tfile.close()
+
+        # Create a temp folder
+        dstpath = tempfile.mkdtemp(prefix="keybase")
+
+        # Set the destination filename
+        srcpath_basename = os.path.basename(srcpath)
+        dstfile = os.path.join(dstpath,
+                               srcpath_basename,
+                               os.path.basename(srcfile))
+
+        # Make sure the source file and destination folder exist and the
+        # destination file doesn't yet exist
+        assert os.path.isdir(srcpath)
+        assert os.path.isfile(srcfile)
+        assert os.path.isdir(dstpath)
+        assert not os.path.exists(dstfile)
+
+        # Trigger copy, and...
+        utils.copy(srcpath, dstfile)
+
+        # ...expect the Keybase CLI to have been invoked. The call was stubbed
+        # so no files were actually copied.
+        expected_args = ['keybase', 'fs', 'cp', srcpath, dstfile]
+        subprocess.check_output.assert_called_with(expected_args)
+
+        # Let's clean up
+        utils.delete(srcpath)
+        utils.delete(dstpath)
+
     def test_link_file(self):
         # Create a tmp file
         tfile = tempfile.NamedTemporaryFile(delete=False)
@@ -335,3 +402,29 @@ def test_can_file_be_synced_on_current_platform(self):
         # Try to use the library path on Linux, which shouldn't work
         path = os.path.join(os.environ["HOME"], "Library/")
         assert not utils.can_file_be_synced_on_current_platform(path)
+
+    def test_get_keybase_folder_location(self):
+        username = str(random.randint(0, 10000))
+        actual = utils.get_keybase_folder_location(username)
+        assert actual == '/keybase/private/{}'.format(username)
+
+    @patch.object(utils, 'CONFIG', Mock())
+    def test_can_remove_acl_with_current_engine_with_dropbox(self):
+        utils.CONFIG.engine = 'dropbox'
+        assert utils.can_remove_acl_with_current_engine() == True
+
+    @patch.object(utils, 'CONFIG', Mock())
+    def test_can_remove_acl_with_current_engine_with_keybase(self):
+        utils.CONFIG.engine = 'keybase'
+        assert utils.can_remove_acl_with_current_engine() == False
+
+    @patch.object(utils, 'CONFIG', None)
+    def test_current_engine_no_config(self):
+        """current_engine returns None if no CONFIG is present"""
+        assert utils.current_engine() is None
+
+    @patch.object(utils, 'CONFIG', Mock())
+    def test_current_engine(self):
+        engine = str(random.randint(0, 1000))
+        utils.CONFIG.engine = engine
+        assert utils.current_engine() is engine