diff --git a/app/src/onedrive_client/utils/constants.py b/app/src/onedrive_client/utils/constants.py new file mode 100644 index 0000000..62bcd8c --- /dev/null +++ b/app/src/onedrive_client/utils/constants.py @@ -0,0 +1,18 @@ +""" +Common constants +""" + +ACTIONS = { + 'start': 'start the service', + 'restart': 'stop and restart the service if the service ' + 'is already running, otherwise start the service', + 'stop': 'stop the service', + 'try-restart': 'restart the service if the service ' + 'is already running', + 'reload': 'cause the configuration of the service to be reloaded ' + 'without actually stopping and restarting the service', + 'force-reload': 'cause the configuration to be reloaded if the ' + 'service supports this, otherwise restart the ' + 'service if it is running', + 'status': 'print the current status of the service' +} diff --git a/app/src/onedrive_client/utils/daemon.py b/app/src/onedrive_client/utils/daemon.py index 0df8635..43f137a 100644 --- a/app/src/onedrive_client/utils/daemon.py +++ b/app/src/onedrive_client/utils/daemon.py @@ -49,9 +49,9 @@ def stop(self, force=True): pid = self.get_pid() if not pid: - message = 'pidfile %s does not exist. ' + \ - 'Daemon not running?\n' - print(message, self.pid) + message = 'pidfile %s does not exist. '\ + 'Daemon not running?\n' % str(self.pid) + print(message) # not an error in a restart if force: return 0 diff --git a/entities/proto/onedrive_client/entities/common.proto b/entities/proto/onedrive_client/entities/common.proto index 90f3dcb..68a18f5 100644 --- a/entities/proto/onedrive_client/entities/common.proto +++ b/entities/proto/onedrive_client/entities/common.proto @@ -16,6 +16,8 @@ import "onedrive_client/entities/onedrive.proto"; message LocalItemMetadata { uint64 size = 1; uint64 modified = 2; + bool is_dir = 3; + bool is_deleted = 4; } diff --git a/entities/python/src/onedrive_client/entities/base.py b/entities/python/src/onedrive_client/entities/base.py index 90d20ea..e65d815 100644 --- a/entities/python/src/onedrive_client/entities/base.py +++ b/entities/python/src/onedrive_client/entities/base.py @@ -165,11 +165,14 @@ class Entity(collections.abc.MutableMapping, metaclass=_EntityMeta): Complex fields are automatically converted to instances of the corresponding entity-classes: - >>> local_item['metadata'] = {'size': 999, 'modified': 123456789} + >>> local_item['metadata'] = {'size': 999, 'modified': 123456789,\ + 'is_dir': True} >>> local_item LocalItem({'path': '/bar', 'metadata': LocalItemMetadata({'size': 999, - 'modified': 123456789})}) + 'modified': 123456789, + 'is_dir': True, + 'is_deleted': False})}) Repeated fields. Notice repeated fields are set by default unlike complex fields. It's because it's Protobuf specs: diff --git a/entities/python/src/onedrive_client/entities/common_pb2.py b/entities/python/src/onedrive_client/entities/common_pb2.py index 00d5f2e..1e5b862 100644 --- a/entities/python/src/onedrive_client/entities/common_pb2.py +++ b/entities/python/src/onedrive_client/entities/common_pb2.py @@ -20,7 +20,7 @@ name='onedrive_client/entities/common.proto', package='onedrive_client.entities.common', syntax='proto3', - serialized_pb=_b('\n%onedrive_client/entities/common.proto\x12\x1fonedrive_client.entities.common\x1a\'onedrive_client/entities/onedrive.proto\"3\n\x11LocalItemMetadata\x12\x0c\n\x04size\x18\x01 \x01(\x04\x12\x10\n\x08modified\x18\x02 \x01(\x04\"_\n\tLocalItem\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x44\n\x08metadata\x18\x02 \x01(\x0b\x32\x32.onedrive_client.entities.common.LocalItemMetadata\"Z\n\x0e\x44irtyLocalItem\x12\x0e\n\x06our_id\x18\x01 \x01(\x0c\x12\x38\n\x04item\x18\x02 \x01(\x0b\x32*.onedrive_client.entities.common.LocalItem\"X\n\x0f\x44irtyRemoteItem\x12\x0e\n\x06our_id\x18\x01 \x01(\x0c\x12\x35\n\x04item\x18\x02 \x01(\x0b\x32\'.onedrive_client.entities.onedrive.Itemb\x06proto3') + serialized_pb=_b('\n%onedrive_client/entities/common.proto\x12\x1fonedrive_client.entities.common\x1a\'onedrive_client/entities/onedrive.proto\"W\n\x11LocalItemMetadata\x12\x0c\n\x04size\x18\x01 \x01(\x04\x12\x10\n\x08modified\x18\x02 \x01(\x04\x12\x0e\n\x06is_dir\x18\x03 \x01(\x08\x12\x12\n\nis_deleted\x18\x04 \x01(\x08\"_\n\tLocalItem\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x44\n\x08metadata\x18\x02 \x01(\x0b\x32\x32.onedrive_client.entities.common.LocalItemMetadata\"Z\n\x0e\x44irtyLocalItem\x12\x0e\n\x06our_id\x18\x01 \x01(\x0c\x12\x38\n\x04item\x18\x02 \x01(\x0b\x32*.onedrive_client.entities.common.LocalItem\"X\n\x0f\x44irtyRemoteItem\x12\x0e\n\x06our_id\x18\x01 \x01(\x0c\x12\x35\n\x04item\x18\x02 \x01(\x0b\x32\'.onedrive_client.entities.onedrive.Itemb\x06proto3') , dependencies=[onedrive__client_dot_entities_dot_onedrive__pb2.DESCRIPTOR,]) @@ -48,6 +48,20 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + _descriptor.FieldDescriptor( + name='is_dir', full_name='onedrive_client.entities.common.LocalItemMetadata.is_dir', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='is_deleted', full_name='onedrive_client.entities.common.LocalItemMetadata.is_deleted', index=3, + number=4, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -61,7 +75,7 @@ oneofs=[ ], serialized_start=115, - serialized_end=166, + serialized_end=202, ) @@ -98,8 +112,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=168, - serialized_end=263, + serialized_start=204, + serialized_end=299, ) @@ -136,8 +150,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=265, - serialized_end=355, + serialized_start=301, + serialized_end=391, ) @@ -174,8 +188,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=357, - serialized_end=445, + serialized_start=393, + serialized_end=481, ) _LOCALITEM.fields_by_name['metadata'].message_type = _LOCALITEMMETADATA diff --git a/filesystem_service/src/filesystem_service/fsmonitor b/filesystem_service/src/filesystem_service/fsmonitor new file mode 100755 index 0000000..d2a178b --- /dev/null +++ b/filesystem_service/src/filesystem_service/fsmonitor @@ -0,0 +1,93 @@ +#!/usr/bin/python + +""" +Service module for the filesystem monitoring +""" + +import argparse +from pathlib import Path + +from filesystem_service.monitor import FileSystemMonitor +from onedrive_client.utils.constants import ACTIONS +from onedrive_client.utils.daemon import Daemon + +_FILESYSTEM_MONITOR = 'fsmonitor' +_FILESYSTEM_MONITOR_VERSION = '0.1.0' +_CONFIG_PATH = '/etc/fsmonitor.yml' +_PID_FILE = '/var/run/fsmonitor.pid' + + +# pylint: disable=bare-except +# pylint: disable=too-few-public-methods +class Monitor(object): + """ + Monitoring starter + """ + + def __init__(self): + try: + self.config = load_config() + # ToDo: add proper exception + except: + print('Failed to load config') + exit(1) + + def start(self): + """ + Starts monitoring of file system events + """ + + mon = FileSystemMonitor(self.config.get('watch', str(Path.home()))) + # ToDo: add subscribers + for exclude in self.config.get('exclude', []): + mon.add_exclude_folder(exclude) + mon.monitor() + + +def load_config(): + """ + :return: + """ + + return dict() + + +def main(): + """ + :return: + """ + + desc = 'Starts monitoring of specified local folder' + epilog = 'OneDrive-L Filesystem Monitoring Service' + + parser = argparse.ArgumentParser(prog='filesystem-monitor', + description=desc, epilog=epilog) + parser.add_argument('-v', '--version', dest='version', action='version', + version='%(prog)s ' + _FILESYSTEM_MONITOR_VERSION, + help='Display version information') + + subparsers = parser.add_subparsers(title='Commands') + for action, help_msg in ACTIONS.items(): + setup = subparsers.add_parser(action, help=help_msg) + setup.set_defaults(which=action) + + args = parser.parse_args() + + fsmonitor = Monitor() + daemon = Daemon(app="filesystem-monitor", pid=_PID_FILE, + action=fsmonitor.start) + + if args.which == 'start': + return daemon.start() + elif args.which == 'stop': + return daemon.stop() + elif args.which in ('restart', 'reload'): + return daemon.restart() + elif args.which in ('try-restart', 'force-reload'): + return daemon.try_restart() + elif args.which == 'status': + return daemon.try_restart() + + +if __name__ == '__main__': + exit(main()) diff --git a/filesystem_service/src/filesystem_service/monitor.py b/filesystem_service/src/filesystem_service/monitor.py index 996c89a..dce5737 100644 --- a/filesystem_service/src/filesystem_service/monitor.py +++ b/filesystem_service/src/filesystem_service/monitor.py @@ -12,6 +12,9 @@ class FileSystemMonitor(object): Monitor local filesystem changes and inform subscribers """ + important_events = ('IN_CLOSE_WRITE', 'IN_CREATE', 'IN_DELETE', + 'IN_MOVED_FROM', 'IN_MOVED_TO', 'IN_ATTRIB') + def __init__(self, folder): """ :param folder: folder to be watched. Type: string @@ -21,6 +24,7 @@ def __init__(self, folder): self._exclude_folders = set() self._subscribers = set() self._notifier = inotify.adapters.Inotify() + self._move_from_cookies = dict() @property def subscribers(self): @@ -82,6 +86,41 @@ def __remove_watch(self, folder): self._notifier.remove_watch(folder.encode(), superficial=True) + @staticmethod + def __is_event_important(event): + """ + :param event: inotify event + :return: True if any event type in inotify event is in + FileSystemMonitor.important_events list + """ + + if any(e_type in FileSystemMonitor.important_events + for e_type in event[1]): + return True + return False + + def __gen_events_on_move_to(self, move_to_event): + """ + :param move_to_event: Event object generated for the MOVE_TO event type + Notifies subscribers with two events: the first one is about to remove + old file, the second is about to create new file + """ + + if move_to_event.cookie in self._move_from_cookies: + move_from_event = self._move_from_cookies[move_to_event.cookie] + move_from_event.in_delete = True + self.__notify(move_from_event, move_to_event) + + def __notify(self, *events): + """ + :param events: event object(s) + Notifies all subscribers about events + """ + + for subscriber in self._subscribers: + for event in events: + subscriber.update(event) + def monitor(self): """ Notify subscribers about filesystem events @@ -92,19 +131,31 @@ def monitor(self): self._notifier.add_watch(folder) for event in self._notifier.event_gen(): - if event: - for subscriber in self._subscribers: - file_object = Event(event) - subscriber.update(file_object) + if event and FileSystemMonitor.__is_event_important(event): + file_object = Event(event) + + # Saving event object which is being moved in dictionary to + if hasattr(file_object, 'IN_MOVED_FROM'): + if file_object.cookie not in self._move_from_cookies: + self._move_from_cookies[file_object.cookie] = \ + file_object + continue + + # Generating appropriate events on file movement + if hasattr(file_object, 'IN_MOVED_TO'): + self.__gen_events_on_move_to(file_object) + continue + + # Removing a watch on folder 'cause it was deleted + if hasattr(file_object, 'IN_DELETE'): + self.__remove_watch(file_object.file_path) - # Removing a watch on folder 'cause it was deleted - if hasattr(file_object, 'IN_DELETE'): - self.__remove_watch(file_object.file_path) + # Adding a watch on folder 'cause we're being recursive + if hasattr(file_object, 'IN_CREATE') and \ + os.path.isdir(file_object.file_path): + self.__add_watch(file_object.file_path) - # Adding a watch on folder 'cause we're being recursive - if hasattr(file_object, 'IN_CREATE') and \ - os.path.isdir(file_object.file_path): - self.__add_watch(file_object.file_path) + self.__notify(file_object) # pylint: disable=too-few-public-methods, too-many-instance-attributes @@ -126,3 +177,5 @@ def __init__(self, event): for i_event in type_names: if not hasattr(self, i_event): setattr(self, i_event, True) + self.is_dir = os.path.isdir(self.file_path) + self.in_delete = True if hasattr(self, 'IN_DELETE') else False diff --git a/filesystem_service/tests/test_fsmonitor.py b/filesystem_service/tests/test_fsmonitor.py index 9b02286..4bd3346 100644 --- a/filesystem_service/tests/test_fsmonitor.py +++ b/filesystem_service/tests/test_fsmonitor.py @@ -2,13 +2,18 @@ Module with unittests for the filesystem monitor """ # pylint: disable=protected-access +# pylint: disable=anomalous-backslash-in-string +from multiprocessing import Manager import os +import shutil +from time import sleep import unittest -from .utils.consts import EXCLUDE_FOLDER, FOLDER, TEST_FILE, TEST_SUBDIR -from .utils.monitor import EVENTS, gen_event_object, gen_events_list, \ - get_monitor_instance, SecondSubscriber, SEVENTS, Subscriber +from .utils.consts import EXCLUDE_FOLDER, FOLDER, SUBDIR_COPY_PATH, \ + SUBDIR_FPATH, SUBDIR_PATH, TEST_FILE, TEST_SUBDIR +from .utils.monitor import gen_event_object, get_monitor_instance, \ + SecondSubscriber, start_monitor_process, Subscriber class FileSystemMonitorTest(unittest.TestCase): @@ -75,10 +80,8 @@ def test_add_exclude(self): self.monitor.add_exclude_folder(EXCLUDE_FOLDER) self.assertTrue(self.monitor._exclude_folders, 'List of excludes is empty') - self.assertEqual(len(self.monitor._exclude_folders), 1, 'Lack of excludes in exclude list') - self.assertEqual(list(self.monitor._exclude_folders)[0], EXCLUDE_FOLDER.encode(),) @@ -95,8 +98,7 @@ class EventTest(unittest.TestCase): Tests for the filesystem event class """ - def __init__(self, *args, **kwargs): - super(EventTest, self).__init__(*args, **kwargs) + def setUp(self): self.event = gen_event_object() def test_event_hasattrs(self): @@ -104,7 +106,8 @@ def test_event_hasattrs(self): Verify that all event types are among event object attributes """ - for attr in ['IN_CLOSE_WRITE', 'IN_ACCESS']: + for attr in ['IN_CLOSE_WRITE', 'IN_ACCESS', 'file_path', 'is_dir', + 'in_delete']: self.assertTrue(hasattr(self.event, attr), 'Missing event attributes') @@ -114,10 +117,8 @@ def test_event_attrs(self): """ self.assertEqual(self.event.cookie, 0, 'Wrong attribute value') - self.assertEqual(self.event.path, '/tmp/onedrive', 'Wrong attribute value') - self.assertEqual(self.event.filename, 'test', 'Wrong attribute value') @@ -126,38 +127,55 @@ class MonitorTest(unittest.TestCase): Tests for the filesystem event's monitoring functionality """ - @classmethod - def setUpClass(cls): - if not os.path.exists(FOLDER): - os.mkdir(FOLDER) - gen_events_list(get_monitor_instance(FOLDER, [Subscriber(), - SecondSubscriber()])) + def setUp(self): + if os.path.exists(FOLDER): + shutil.rmtree(FOLDER) + os.makedirs(FOLDER) + self.events = Manager().list() + self.sevents = Manager().list() + self.process = start_monitor_process( + get_monitor_instance(FOLDER, [Subscriber(self.events), + SecondSubscriber(self.sevents)])) + + def tearDown(self): + self.process.terminate() + sleep(0.1) def test_empty_events(self): """ Check if events list is not empty """ - self.assertTrue(EVENTS, 'List of events is empty for the ' - 'first subscriber') - self.assertTrue(SEVENTS, 'List of events is empty for the ' - 'second subscriber') + os.system('mkdir -p {}'.format(SUBDIR_PATH)) + sleep(1) + + self.assertTrue(self.events, 'List of events is empty for the ' + 'first subscriber') + self.assertTrue(self.sevents, 'List of events is empty for the ' + 'second subscriber') def test_event_lists(self): """ Check if event lists are equal between subscribers """ - self.assertFalse(list(set(EVENTS) & set(SEVENTS)), + os.system('mkdir -p {}'.format(SUBDIR_PATH)) + sleep(1) + + self.assertFalse(list(set(self.events) & set(self.sevents)), 'List of events for subscribers are not equal') def test_event_create(self): """ - Test create events + Test handling of create events """ + os.system('mkdir -p {}'.format(SUBDIR_PATH)) + os.system('touch {}'.format(SUBDIR_FPATH)) + sleep(1) + create_events = [] - for event in EVENTS: + for event in self.events: if hasattr(event, 'IN_CREATE'): create_events.append(event) @@ -171,18 +189,85 @@ def test_event_create(self): self.assertTrue(TEST_FILE in files_created, 'Create event for {} is missing'.format(TEST_FILE)) + def test_event_copy(self): + """ + Test handling of copy events + """ + + os.system('mkdir -p {}'.format(SUBDIR_PATH)) + os.system('touch {}'.format(SUBDIR_FPATH)) + os.system('\cp -R {} {}'.format(SUBDIR_PATH, SUBDIR_COPY_PATH)) + os.system('echo "test" > {}'.format(os.path.join(SUBDIR_COPY_PATH, + TEST_FILE))) + sleep(1) + + create_events = [] + read_event = False + for event in self.events: + if hasattr(event, 'IN_CREATE'): + create_events.append(event) + if event.filename == TEST_FILE and \ + hasattr(event, 'IN_CLOSE_WRITE'): + read_event = True + + files_created = [event.filename for event in create_events] + + self.assertTrue(create_events, 'Create events were not found') + self.assertEqual(len(files_created), 3, + 'Wrong number of create events') + self.assertTrue(read_event, + '{} is not monitored after recursive copying' + .format(SUBDIR_COPY_PATH)) + def test_event_delete(self): """ Test delete events """ + os.system('touch {}'.format(os.path.join(FOLDER, TEST_FILE))) + os.system('rm -f {}'.format(os.path.join(FOLDER, TEST_FILE))) + sleep(1) + delete_event = None - for event in EVENTS: - if hasattr(event, 'IN_DELETE') and event.filename == TEST_SUBDIR: + for event in self.events: + if hasattr(event, 'IN_DELETE') and event.filename == TEST_FILE: delete_event = event break - self.assertIsNotNone(delete_event, 'Delete events were not found') + self.assertIsNotNone(delete_event, 'Delete event was not found') + self.assertEqual(delete_event.filename, TEST_FILE, + 'Delete event for {} is missing' + .format(os.path.join(FOLDER, TEST_FILE))) + + def test_event_move(self): + """ + Test move events + """ - self.assertEqual(delete_event.filename, TEST_SUBDIR, - 'Delete event for {} is missing'.format(TEST_SUBDIR)) + mfilename = 'm' + TEST_FILE + tfilepath = os.path.join(FOLDER, TEST_FILE) + mfilepath = os.path.join(FOLDER, mfilename) + os.system('touch {}'.format(tfilepath)) + os.system('mv {} {}'.format(tfilepath, mfilepath)) + sleep(1) + + moved_from_event = None + moved_to_event = None + for event in self.events: + if hasattr(event, 'IN_MOVED_FROM') and event.filename == TEST_FILE: + moved_from_event = event + if hasattr(event, 'IN_MOVED_TO') and event.filename == mfilename: + moved_to_event = event + + self.assertIsNotNone(moved_from_event, + 'MOVED_FROM event was not found') + self.assertIsNotNone(moved_to_event, 'MOVED_TO event was not found') + self.assertEqual(moved_from_event.filename, TEST_FILE, + 'Delete event for {} is missing' + .format(tfilepath)) + self.assertEqual(moved_to_event.filename, mfilename, + 'Delete event for {} is missing' + .format(mfilepath)) + self.assertTrue(moved_from_event.in_delete, + 'in_delete attribute of the MOVED_FROM event ' + 'was no set properly') diff --git a/filesystem_service/tests/utils/consts.py b/filesystem_service/tests/utils/consts.py index 87c0594..eebb8ec 100644 --- a/filesystem_service/tests/utils/consts.py +++ b/filesystem_service/tests/utils/consts.py @@ -10,4 +10,5 @@ TEST_FILE = 'testfile' TEST_SUBDIR = 'subdir' SUBDIR_PATH = os.path.join(FOLDER, TEST_SUBDIR) +SUBDIR_COPY_PATH = os.path.join(FOLDER, 'subdir_copy') SUBDIR_FPATH = os.path.join(SUBDIR_PATH, TEST_FILE) diff --git a/filesystem_service/tests/utils/monitor.py b/filesystem_service/tests/utils/monitor.py index a76d456..48f4f45 100644 --- a/filesystem_service/tests/utils/monitor.py +++ b/filesystem_service/tests/utils/monitor.py @@ -3,35 +3,33 @@ """ import collections -from multiprocessing import Manager, Process -import os +from multiprocessing import Process from time import sleep from filesystem_service.monitor import Event, FileSystemMonitor -from .consts import SUBDIR_FPATH, SUBDIR_PATH - InotifyEvent = collections.namedtuple('_INOTIFY_EVENT', ['wd', 'mask', 'cookie', 'len']) -EVENTS = Manager().list() -SEVENTS = Manager().list() +# pylint: disable=dangerous-default-value # pylint: disable=too-few-public-methods class Subscriber(object): """ Subscriber emulator """ - @staticmethod - def update(event): + def __init__(self, events=[]): + self.events = events + + def update(self, event): """ :param event: inotify event add events to shared list variable """ - EVENTS.append(event) + self.events.append(event) class SecondSubscriber(object): @@ -39,17 +37,18 @@ class SecondSubscriber(object): Subscriber emulator """ - @staticmethod - def update(event): + def __init__(self, events=[]): + self.events = events + + def update(self, event): """ :param event: inotify event add events to shared list variable """ - SEVENTS.append(event) + self.events.append(event) -# pylint: disable=dangerous-default-value def get_monitor_instance(folder='/test', subscribers=[]): """ :param folder: folder to be watched @@ -87,21 +86,3 @@ def start_monitor_process(monitor): sleep(0.1) return process - - -def gen_events_list(monitor): - """ - :param monitor: an instance of monitor - Fills shared variable with events - """ - - process = start_monitor_process(monitor) - os.system('mkdir -p {}'.format(SUBDIR_PATH)) - sleep(1) - os.system('touch {}'.format(SUBDIR_FPATH)) - sleep(1) - os.system('cat {}'.format(SUBDIR_FPATH)) - sleep(1) - os.system('rm -rf {}'.format(SUBDIR_PATH)) - sleep(1) - process.terminate()