Service file for the filesystem monitor

app/src/onedrive_client/utils/constants.py

@@ -0,0 +1,18 @@
+"""
+Common constants
+"""
+
+ACTIONS = {
+    'start': 'start the service',
+    'restart': 'stop and restart the service if the service '
+               'is already running, otherwise start the service',
+    'stop': 'stop the service',
+    'try-restart': 'restart the service if the service '
+                   'is already running',
+    'reload': 'cause the configuration of the service to be reloaded '
+              'without actually stopping and restarting the service',
+    'force-reload': 'cause the configuration to be reloaded if the '
+                    'service supports this, otherwise restart the '
+                    'service if it is running',
+    'status': 'print the current status of the service'
+}

app/src/onedrive_client/utils/daemon.py

@@ -49,9 +49,9 @@ def stop(self, force=True):
 
         pid = self.get_pid()
         if not pid:
-            message = 'pidfile %s does not exist. ' + \
-                      'Daemon not running?\n'
-            print(message, self.pid)
+            message = 'pidfile %s does not exist. '\
+                      'Daemon not running?\n' % str(self.pid)
+            print(message)
             # not an error in a restart
             if force:
                 return 0

entities/proto/onedrive_client/entities/common.proto

@@ -16,6 +16,8 @@ import "onedrive_client/entities/onedrive.proto";
 message LocalItemMetadata {
   uint64 size = 1;
   uint64 modified = 2;
+  bool is_dir = 3;
+  bool is_deleted = 4;
 }
 
 

entities/python/src/onedrive_client/entities/base.py

@@ -165,11 +165,14 @@ class Entity(collections.abc.MutableMapping, metaclass=_EntityMeta):
 
     Complex fields are automatically converted to instances of
     the corresponding entity-classes:
-    >>> local_item['metadata'] = {'size': 999, 'modified': 123456789}
+    >>> local_item['metadata'] = {'size': 999, 'modified': 123456789,\
+                                  'is_dir': True}
     >>> local_item
     LocalItem({'path': '/bar',
                'metadata': LocalItemMetadata({'size': 999,
-                                              'modified': 123456789})})
+                                              'modified': 123456789,
+                                              'is_dir': True,
+                                              'is_deleted': False})})
 
     Repeated fields. Notice repeated fields are set by default unlike
     complex fields. It's because it's Protobuf specs:

filesystem_service/src/filesystem_service/fsmonitor

@@ -0,0 +1,93 @@
+#!/usr/bin/python
+
+"""
+Service module for the filesystem monitoring
+"""
+
+import argparse
+from pathlib import Path
+
+from filesystem_service.monitor import FileSystemMonitor
+from onedrive_client.utils.constants import ACTIONS
+from onedrive_client.utils.daemon import Daemon
+
+_FILESYSTEM_MONITOR = 'fsmonitor'
+_FILESYSTEM_MONITOR_VERSION = '0.1.0'
+_CONFIG_PATH = '/etc/fsmonitor.yml'
+_PID_FILE = '/var/run/fsmonitor.pid'
+
+
+# pylint: disable=bare-except
+# pylint: disable=too-few-public-methods
+class Monitor(object):
+    """
+    Monitoring starter
+    """
+
+    def __init__(self):
+        try:
+            self.config = load_config()
+        # ToDo: add proper exception
+        except:
+            print('Failed to load config')
+            exit(1)
+
+    def start(self):
+        """
+        Starts monitoring of file system events
+        """
+
+        mon = FileSystemMonitor(self.config.get('watch', str(Path.home())))
+        # ToDo: add subscribers
+        for exclude in self.config.get('exclude', []):
+            mon.add_exclude_folder(exclude)
+        mon.monitor()
+
+
+def load_config():
+    """
+    :return:
+    """
+
+    return dict()
+
+
+def main():
+    """
+    :return:
+    """
+
+    desc = 'Starts monitoring of specified local folder'
+    epilog = 'OneDrive-L Filesystem Monitoring Service'
+
+    parser = argparse.ArgumentParser(prog='filesystem-monitor',
+                                     description=desc, epilog=epilog)
+    parser.add_argument('-v', '--version', dest='version', action='version',
+                        version='%(prog)s ' + _FILESYSTEM_MONITOR_VERSION,
+                        help='Display version information')
+
+    subparsers = parser.add_subparsers(title='Commands')
+    for action, help_msg in ACTIONS.items():
+        setup = subparsers.add_parser(action, help=help_msg)
+        setup.set_defaults(which=action)
+
+    args = parser.parse_args()
+
+    fsmonitor = Monitor()
+    daemon = Daemon(app="filesystem-monitor", pid=_PID_FILE,
+                    action=fsmonitor.start)
+
+    if args.which == 'start':
+        return daemon.start()
+    elif args.which == 'stop':
+        return daemon.stop()
+    elif args.which in ('restart', 'reload'):
+        return daemon.restart()
+    elif args.which in ('try-restart', 'force-reload'):
+        return daemon.try_restart()
+    elif args.which == 'status':
+        return daemon.try_restart()
+
+
+if __name__ == '__main__':
+    exit(main())

filesystem_service/src/filesystem_service/monitor.py

@@ -12,6 +12,9 @@ class FileSystemMonitor(object):
     Monitor local filesystem changes and inform subscribers
     """
 
+    important_events = ('IN_CLOSE_WRITE', 'IN_CREATE', 'IN_DELETE',
+                        'IN_MOVED_FROM', 'IN_MOVED_TO', 'IN_ATTRIB')
+
     def __init__(self, folder):
         """
         :param folder: folder to be watched. Type: string
@@ -21,6 +24,7 @@ def __init__(self, folder):
         self._exclude_folders = set()
         self._subscribers = set()
         self._notifier = inotify.adapters.Inotify()
+        self._move_from_cookies = dict()
 
     @property
     def subscribers(self):
@@ -82,6 +86,41 @@ def __remove_watch(self, folder):
 
         self._notifier.remove_watch(folder.encode(), superficial=True)
 
+    @staticmethod
+    def __is_event_important(event):
+        """
+        :param event: inotify event
+        :return: True if any event type in inotify event is in
+        FileSystemMonitor.important_events list
+        """
+
+        if any(e_type in FileSystemMonitor.important_events
+               for e_type in event[1]):
+            return True
+        return False
+
+    def __gen_events_on_move_to(self, move_to_event):
+        """
+        :param move_to_event: Event object generated for the MOVE_TO event type
+        Notifies subscribers with two events: the first one is about to remove
+        old file, the second is about to create new file
+        """
+
+        if move_to_event.cookie in self._move_from_cookies:
+            move_from_event = self._move_from_cookies[move_to_event.cookie]
+            move_from_event.in_delete = True
+            self.__notify(move_from_event, move_to_event)
+
+    def __notify(self, *events):
+        """
+        :param events: event object(s)
+        Notifies all subscribers about events
+        """
+
+        for subscriber in self._subscribers:
+            for event in events:
+                subscriber.update(event)
+
     def monitor(self):
         """
         Notify subscribers about filesystem events
@@ -92,19 +131,31 @@ def monitor(self):
             self._notifier.add_watch(folder)
 
         for event in self._notifier.event_gen():
-            if event:
-                for subscriber in self._subscribers:
-                    file_object = Event(event)
-                    subscriber.update(file_object)
+            if event and FileSystemMonitor.__is_event_important(event):
+                file_object = Event(event)
+
+                # Saving event object which is being moved in dictionary to
+                if hasattr(file_object, 'IN_MOVED_FROM'):
+                    if file_object.cookie not in self._move_from_cookies:
+                        self._move_from_cookies[file_object.cookie] = \
+                            file_object
+                        continue
+
+                # Generating appropriate events on file movement
+                if hasattr(file_object, 'IN_MOVED_TO'):
+                    self.__gen_events_on_move_to(file_object)
+                    continue
+
+                # Removing a watch on folder 'cause it was deleted
+                if hasattr(file_object, 'IN_DELETE'):
+                    self.__remove_watch(file_object.file_path)
 
-                    # Removing a watch on folder 'cause it was deleted
-                    if hasattr(file_object, 'IN_DELETE'):
-                        self.__remove_watch(file_object.file_path)
+                # Adding a watch on folder 'cause we're being recursive
+                if hasattr(file_object, 'IN_CREATE') and \
+                        os.path.isdir(file_object.file_path):
+                    self.__add_watch(file_object.file_path)
 
-                    # Adding a watch on folder 'cause we're being recursive
-                    if hasattr(file_object, 'IN_CREATE') and \
-                            os.path.isdir(file_object.file_path):
-                        self.__add_watch(file_object.file_path)
+                self.__notify(file_object)
 
 
 # pylint: disable=too-few-public-methods, too-many-instance-attributes
@@ -126,3 +177,5 @@ def __init__(self, event):
         for i_event in type_names:
             if not hasattr(self, i_event):
                 setattr(self, i_event, True)
+        self.is_dir = os.path.isdir(self.file_path)
+        self.in_delete = True if hasattr(self, 'IN_DELETE') else False

filesystem_service/tests/test_fsmonitor.py

@@ -2,13 +2,18 @@
 Module with unittests for the filesystem monitor
 """
 # pylint: disable=protected-access
+# pylint: disable=anomalous-backslash-in-string
 
+from multiprocessing import Manager
 import os
+import shutil
+from time import sleep
 import unittest
 
-from .utils.consts import EXCLUDE_FOLDER, FOLDER, TEST_FILE, TEST_SUBDIR
-from .utils.monitor import EVENTS, gen_event_object, gen_events_list, \
-    get_monitor_instance, SecondSubscriber, SEVENTS, Subscriber
+from .utils.consts import EXCLUDE_FOLDER, FOLDER, SUBDIR_COPY_PATH, \
+    SUBDIR_FPATH, SUBDIR_PATH, TEST_FILE, TEST_SUBDIR
+from .utils.monitor import gen_event_object, get_monitor_instance, \
+    SecondSubscriber, start_monitor_process, Subscriber
 
 
 class FileSystemMonitorTest(unittest.TestCase):
@@ -75,10 +80,8 @@ def test_add_exclude(self):
         self.monitor.add_exclude_folder(EXCLUDE_FOLDER)
         self.assertTrue(self.monitor._exclude_folders,
                         'List of excludes is empty')
-
         self.assertEqual(len(self.monitor._exclude_folders), 1,
                          'Lack of excludes in exclude list')
-
         self.assertEqual(list(self.monitor._exclude_folders)[0],
                          EXCLUDE_FOLDER.encode(),)
 
@@ -95,16 +98,16 @@ class EventTest(unittest.TestCase):
     Tests for the filesystem event class
     """
 
-    def __init__(self, *args, **kwargs):
-        super(EventTest, self).__init__(*args, **kwargs)
+    def setUp(self):
         self.event = gen_event_object()
 
     def test_event_hasattrs(self):
         """
         Verify that all event types are among event object attributes
         """
 
-        for attr in ['IN_CLOSE_WRITE', 'IN_ACCESS']:
+        for attr in ['IN_CLOSE_WRITE', 'IN_ACCESS', 'file_path', 'is_dir',
+                     'in_delete']:
             self.assertTrue(hasattr(self.event, attr),
                             'Missing event attributes')
 
@@ -114,10 +117,8 @@ def test_event_attrs(self):
         """
 
         self.assertEqual(self.event.cookie, 0, 'Wrong attribute value')
-
         self.assertEqual(self.event.path, '/tmp/onedrive',
                          'Wrong attribute value')
-
         self.assertEqual(self.event.filename, 'test', 'Wrong attribute value')
 
 
@@ -126,38 +127,55 @@ class MonitorTest(unittest.TestCase):
     Tests for the filesystem event's monitoring functionality
     """
 
-    @classmethod
-    def setUpClass(cls):
-        if not os.path.exists(FOLDER):
-            os.mkdir(FOLDER)
-        gen_events_list(get_monitor_instance(FOLDER, [Subscriber(),
-                                                      SecondSubscriber()]))
+    def setUp(self):
+        if os.path.exists(FOLDER):
+            shutil.rmtree(FOLDER)
+        os.makedirs(FOLDER)
+        self.events = Manager().list()
+        self.sevents = Manager().list()
+        self.process = start_monitor_process(
+            get_monitor_instance(FOLDER, [Subscriber(self.events),
+                                          SecondSubscriber(self.sevents)]))
+
+    def tearDown(self):
+        self.process.terminate()
+        sleep(0.1)
 
     def test_empty_events(self):
         """
         Check if events list is not empty
         """
 
-        self.assertTrue(EVENTS, 'List of events is empty for the '
-                                'first subscriber')
-        self.assertTrue(SEVENTS, 'List of events is empty for the '
-                                 'second subscriber')
+        os.system('mkdir -p {}'.format(SUBDIR_PATH))
+        sleep(1)
+
+        self.assertTrue(self.events, 'List of events is empty for the '
+                                     'first subscriber')
+        self.assertTrue(self.sevents, 'List of events is empty for the '
+                                      'second subscriber')
 
     def test_event_lists(self):
         """
         Check if event lists are equal between subscribers
         """
 
-        self.assertFalse(list(set(EVENTS) & set(SEVENTS)),
+        os.system('mkdir -p {}'.format(SUBDIR_PATH))
+        sleep(1)
+
+        self.assertFalse(list(set(self.events) & set(self.sevents)),
                          'List of events for subscribers are not equal')
 
     def test_event_create(self):
         """
-        Test create events
+        Test handling of create events
         """
 
+        os.system('mkdir -p {}'.format(SUBDIR_PATH))
+        os.system('touch {}'.format(SUBDIR_FPATH))
+        sleep(1)
+
         create_events = []
-        for event in EVENTS:
+        for event in self.events:
             if hasattr(event, 'IN_CREATE'):
                 create_events.append(event)
 
@@ -171,18 +189,85 @@ def test_event_create(self):
         self.assertTrue(TEST_FILE in files_created,
                         'Create event for {} is missing'.format(TEST_FILE))
 
+    def test_event_copy(self):
+        """
+         Test handling of copy events
+        """
+
+        os.system('mkdir -p {}'.format(SUBDIR_PATH))
+        os.system('touch {}'.format(SUBDIR_FPATH))
+        os.system('\cp -R {} {}'.format(SUBDIR_PATH, SUBDIR_COPY_PATH))
+        os.system('echo "test"  > {}'.format(os.path.join(SUBDIR_COPY_PATH,
+                                                          TEST_FILE)))
+        sleep(1)
+
+        create_events = []
+        read_event = False
+        for event in self.events:
+            if hasattr(event, 'IN_CREATE'):
+                create_events.append(event)
+            if event.filename == TEST_FILE and \
+                    hasattr(event, 'IN_CLOSE_WRITE'):
+                read_event = True
+
+        files_created = [event.filename for event in create_events]
+
+        self.assertTrue(create_events, 'Create events were not found')
+        self.assertEqual(len(files_created), 3,
+                         'Wrong number of create events')
+        self.assertTrue(read_event,
+                        '{} is not monitored after recursive copying'
+                        .format(SUBDIR_COPY_PATH))
+
     def test_event_delete(self):
         """
         Test delete events
         """
 
+        os.system('touch {}'.format(os.path.join(FOLDER, TEST_FILE)))
+        os.system('rm -f {}'.format(os.path.join(FOLDER, TEST_FILE)))
+        sleep(1)
+
         delete_event = None
-        for event in EVENTS:
-            if hasattr(event, 'IN_DELETE') and event.filename == TEST_SUBDIR:
+        for event in self.events:
+            if hasattr(event, 'IN_DELETE') and event.filename == TEST_FILE:
                 delete_event = event
                 break
 
-        self.assertIsNotNone(delete_event, 'Delete events were not found')
+        self.assertIsNotNone(delete_event, 'Delete event was not found')
+        self.assertEqual(delete_event.filename, TEST_FILE,
+                         'Delete event for {} is missing'
+                         .format(os.path.join(FOLDER, TEST_FILE)))
+
+    def test_event_move(self):
+        """
+        Test move events
+        """
 
-        self.assertEqual(delete_event.filename, TEST_SUBDIR,
-                         'Delete event for {} is missing'.format(TEST_SUBDIR))
+        mfilename = 'm' + TEST_FILE
+        tfilepath = os.path.join(FOLDER, TEST_FILE)
+        mfilepath = os.path.join(FOLDER, mfilename)
+        os.system('touch {}'.format(tfilepath))
+        os.system('mv {} {}'.format(tfilepath, mfilepath))
+        sleep(1)
+
+        moved_from_event = None
+        moved_to_event = None
+        for event in self.events:
+            if hasattr(event, 'IN_MOVED_FROM') and event.filename == TEST_FILE:
+                moved_from_event = event
+            if hasattr(event, 'IN_MOVED_TO') and event.filename == mfilename:
+                moved_to_event = event
+
+        self.assertIsNotNone(moved_from_event,
+                             'MOVED_FROM event was not found')
+        self.assertIsNotNone(moved_to_event, 'MOVED_TO event was not found')
+        self.assertEqual(moved_from_event.filename, TEST_FILE,
+                         'Delete event for {} is missing'
+                         .format(tfilepath))
+        self.assertEqual(moved_to_event.filename, mfilename,
+                         'Delete event for {} is missing'
+                         .format(mfilepath))
+        self.assertTrue(moved_from_event.in_delete,
+                        'in_delete attribute of the MOVED_FROM event '
+                        'was no set properly')

filesystem_service/tests/utils/consts.py

@@ -10,4 +10,5 @@
 TEST_FILE = 'testfile'
 TEST_SUBDIR = 'subdir'
 SUBDIR_PATH = os.path.join(FOLDER, TEST_SUBDIR)
+SUBDIR_COPY_PATH = os.path.join(FOLDER, 'subdir_copy')
 SUBDIR_FPATH = os.path.join(SUBDIR_PATH, TEST_FILE)

filesystem_service/tests/utils/monitor.py

@@ -3,53 +3,52 @@
 """
 
 import collections
-from multiprocessing import Manager, Process
-import os
+from multiprocessing import Process
 from time import sleep
 
 from filesystem_service.monitor import Event, FileSystemMonitor
 
-from .consts import SUBDIR_FPATH, SUBDIR_PATH
-
 
 InotifyEvent = collections.namedtuple('_INOTIFY_EVENT',
                                       ['wd', 'mask', 'cookie', 'len'])
-EVENTS = Manager().list()
-SEVENTS = Manager().list()
 
 
+# pylint: disable=dangerous-default-value
 # pylint: disable=too-few-public-methods
 class Subscriber(object):
     """
     Subscriber emulator
     """
 
-    @staticmethod
-    def update(event):
+    def __init__(self, events=[]):
+        self.events = events
+
+    def update(self, event):
         """
         :param event: inotify event
         add events to shared list variable
         """
 
-        EVENTS.append(event)
+        self.events.append(event)
 
 
 class SecondSubscriber(object):
     """
     Subscriber emulator
     """
 
-    @staticmethod
-    def update(event):
+    def __init__(self, events=[]):
+        self.events = events
+
+    def update(self, event):
         """
         :param event: inotify event
         add events to shared list variable
         """
 
-        SEVENTS.append(event)
+        self.events.append(event)
 
 
-# pylint: disable=dangerous-default-value
 def get_monitor_instance(folder='/test', subscribers=[]):
     """
     :param folder: folder to be watched
@@ -87,21 +86,3 @@ def start_monitor_process(monitor):
     sleep(0.1)
 
     return process
-
-
-def gen_events_list(monitor):
-    """
-    :param monitor: an instance of monitor
-    Fills shared variable with events
-    """
-
-    process = start_monitor_process(monitor)
-    os.system('mkdir -p {}'.format(SUBDIR_PATH))
-    sleep(1)
-    os.system('touch {}'.format(SUBDIR_FPATH))
-    sleep(1)
-    os.system('cat {}'.format(SUBDIR_FPATH))
-    sleep(1)
-    os.system('rm -rf {}'.format(SUBDIR_PATH))
-    sleep(1)
-    process.terminate()