Simplify package structure

tfw/internals/inotify/__init__.py

@@ -0,0 +1,6 @@
+from .inotify import InotifyObserver
+from .inotify import (
+    InotifyFileCreatedEvent, InotifyFileModifiedEvent, InotifyFileMovedEvent,
+    InotifyFileDeletedEvent, InotifyDirCreatedEvent, InotifyDirModifiedEvent,
+    InotifyDirMovedEvent, InotifyDirDeletedEvent
+)

tfw/internals/inotify/inotify.py

@@ -0,0 +1,189 @@
+# pylint: disable=too-few-public-methods
+
+from typing import Iterable
+from time import time
+from os.path import abspath, dirname, isdir
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemMovedEvent, PatternMatchingEventHandler
+from watchdog.events import (
+    FileCreatedEvent, FileModifiedEvent, FileMovedEvent, FileDeletedEvent,
+    DirCreatedEvent, DirModifiedEvent, DirMovedEvent, DirDeletedEvent
+)
+
+
+class InotifyEvent:
+    def __init__(self, src_path):
+        self.date = time()
+        self.src_path = src_path
+
+    def __str__(self):
+        return self.__repr__()
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}({self.src_path})'
+
+
+class InotifyMovedEvent(InotifyEvent):
+    def __init__(self, src_path, dest_path):
+        self.dest_path = dest_path
+        super().__init__(src_path)
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}({self.src_path}, {self.dest_path})'
+
+
+class InotifyFileCreatedEvent(InotifyEvent):
+    pass
+
+
+class InotifyFileModifiedEvent(InotifyEvent):
+    pass
+
+
+class InotifyFileMovedEvent(InotifyMovedEvent):
+    pass
+
+
+class InotifyFileDeletedEvent(InotifyEvent):
+    pass
+
+
+class InotifyDirCreatedEvent(InotifyEvent):
+    pass
+
+
+class InotifyDirModifiedEvent(InotifyEvent):
+    pass
+
+
+class InotifyDirMovedEvent(InotifyMovedEvent):
+    pass
+
+
+class InotifyDirDeletedEvent(InotifyEvent):
+    pass
+
+
+class InotifyObserver:
+    def __init__(self, path, patterns=None, exclude=None, recursive=False):
+        self._files = []
+        self._paths = path
+        self._patterns = patterns or []
+        self._exclude = exclude
+        self._recursive = recursive
+        self._observer = Observer()
+        self._reset()
+
+    def _reset(self):
+        if isinstance(self._paths, str):
+            self._paths = [self._paths]
+        if isinstance(self._paths, Iterable):
+            self._extract_files_from_paths()
+        else:
+            raise ValueError('Expected one or more string paths.')
+
+        patterns = self._files+self.patterns
+        handler = PatternMatchingEventHandler(patterns if patterns else None, self.exclude)
+        handler.on_any_event = self._dispatch_event
+        self._observer.unschedule_all()
+        for path in self.paths:
+            self._observer.schedule(handler, path, self._recursive)
+
+    def _extract_files_from_paths(self):
+        files, paths = [], []
+        for path in self._paths:
+            path = abspath(path)
+            if isdir(path):
+                paths.append(path)
+            else:
+                paths.append(dirname(path))
+                files.append(path)
+        self._files, self._paths = files, paths
+
+    @property
+    def paths(self):
+        return self._paths
+
+    @paths.setter
+    def paths(self, paths):
+        self._paths = paths
+        self._reset()
+
+    @property
+    def patterns(self):
+        return self._patterns
+
+    @patterns.setter
+    def patterns(self, patterns):
+        self._patterns = patterns or []
+        self._reset()
+
+    @property
+    def exclude(self):
+        return self._exclude
+
+    @exclude.setter
+    def exclude(self, exclude):
+        self._exclude = exclude
+        self._reset()
+
+    def start(self):
+        self._observer.start()
+
+    def stop(self):
+        self._observer.stop()
+        self._observer.join()
+
+    def _dispatch_event(self, event):
+        event_to_action = {
+            InotifyFileCreatedEvent  : self.on_created,
+            InotifyFileModifiedEvent : self.on_modified,
+            InotifyFileMovedEvent    : self.on_moved,
+            InotifyFileDeletedEvent  : self.on_deleted,
+            InotifyDirCreatedEvent   : self.on_created,
+            InotifyDirModifiedEvent  : self.on_modified,
+            InotifyDirMovedEvent     : self.on_moved,
+            InotifyDirDeletedEvent   : self.on_deleted
+        }
+
+        event = self._transform_event(event)
+        self.on_any_event(event)
+        event_to_action[type(event)](event)
+
+    @staticmethod
+    def _transform_event(event):
+        watchdog_to_inotify = {
+            FileCreatedEvent  : InotifyFileCreatedEvent,
+            FileModifiedEvent : InotifyFileModifiedEvent,
+            FileMovedEvent    : InotifyFileMovedEvent,
+            FileDeletedEvent  : InotifyFileDeletedEvent,
+            DirCreatedEvent   : InotifyDirCreatedEvent,
+            DirModifiedEvent  : InotifyDirModifiedEvent,
+            DirMovedEvent     : InotifyDirMovedEvent,
+            DirDeletedEvent   : InotifyDirDeletedEvent
+        }
+
+        try:
+            cls = watchdog_to_inotify[type(event)]
+        except KeyError:
+            raise NameError('Watchdog API returned an unknown event.')
+
+        if isinstance(event, FileSystemMovedEvent):
+            return cls(event.src_path, event.dest_path)
+        return cls(event.src_path)
+
+    def on_any_event(self, event):
+        pass
+
+    def on_created(self, event):
+        pass
+
+    def on_modified(self, event):
+        pass
+
+    def on_moved(self, event):
+        pass
+
+    def on_deleted(self, event):
+        pass

tfw/internals/inotify/test_inotify.py

@@ -0,0 +1,179 @@
+# pylint: disable=redefined-outer-name
+
+from queue import Empty, Queue
+from secrets import token_urlsafe
+from pathlib import Path
+from shutil import rmtree
+from os.path import join
+from os import mkdir, remove, rename
+from tempfile import TemporaryDirectory
+from contextlib import suppress
+
+import watchdog
+import pytest
+
+from .inotify import InotifyObserver
+from .inotify import (
+    InotifyFileCreatedEvent, InotifyFileModifiedEvent, InotifyFileMovedEvent,
+    InotifyFileDeletedEvent, InotifyDirCreatedEvent, InotifyDirModifiedEvent,
+    InotifyDirMovedEvent, InotifyDirDeletedEvent
+)
+
+with suppress(AttributeError):
+    watchdog.observers.inotify_buffer.InotifyBuffer.delay = 0
+
+
+class InotifyContext:
+    def __init__(self, workdir, subdir, subfile, observer):
+        self.missing_events = 0
+        self.workdir = workdir
+        self.subdir = subdir
+        self.subfile = subfile
+        self.observer = observer
+
+        self.event_to_queue = {
+            InotifyFileCreatedEvent  : self.observer.create_queue,
+            InotifyFileModifiedEvent : self.observer.modify_queue,
+            InotifyFileMovedEvent    : self.observer.move_queue,
+            InotifyFileDeletedEvent  : self.observer.delete_queue,
+            InotifyDirCreatedEvent   : self.observer.create_queue,
+            InotifyDirModifiedEvent  : self.observer.modify_queue,
+            InotifyDirMovedEvent     : self.observer.move_queue,
+            InotifyDirDeletedEvent   : self.observer.delete_queue
+        }
+
+    def create_random_file(self, dirname, extension):
+        filename = self.join(f'{dirname}/{generate_name()}{extension}')
+        Path(filename).touch()
+        return filename
+
+    def create_random_folder(self, basepath):
+        dirname = self.join(f'{basepath}/{generate_name()}')
+        mkdir(dirname)
+        return dirname
+
+    def join(self, path):
+        return join(self.workdir, path)
+
+    def check_event(self, event_type, path):
+        self.missing_events += 1
+        event = self.event_to_queue[event_type].get(timeout=0.1)
+        assert isinstance(event, event_type)
+        assert event.src_path == path
+        return event
+
+    def check_empty(self, event_type):
+        with pytest.raises(Empty):
+            self.event_to_queue[event_type].get(timeout=0.1)
+
+    def check_any(self):
+        attrs = self.observer.__dict__.values()
+        total = sum([q.qsize() for q in attrs if isinstance(q, Queue)])
+        return total+self.missing_events == len(self.observer.any_list)
+
+
+class InotifyTestObserver(InotifyObserver):
+    def __init__(self, paths, patterns=None, exclude=None, recursive=False):
+        self.any_list = []
+        self.create_queue, self.modify_queue, self.move_queue, self.delete_queue = [Queue() for _ in range(4)]
+        super().__init__(paths, patterns, exclude, recursive)
+
+    def on_any_event(self, event):
+        self.any_list.append(event)
+
+    def on_created(self, event):
+        self.create_queue.put(event)
+
+    def on_modified(self, event):
+        self.modify_queue.put(event)
+
+    def on_moved(self, event):
+        self.move_queue.put(event)
+
+    def on_deleted(self, event):
+        self.delete_queue.put(event)
+
+def generate_name():
+    return token_urlsafe(16)
+
+@pytest.fixture()
+def context():
+    with TemporaryDirectory() as workdir:
+        subdir = join(workdir, generate_name())
+        subfile = join(subdir, generate_name()+'.txt')
+        mkdir(subdir)
+        Path(subfile).touch()
+        monitor = InotifyTestObserver(workdir, recursive=True)
+        monitor.start()
+        yield InotifyContext(workdir, subdir, subfile, monitor)
+
+def test_create(context):
+    newfile = context.create_random_file(context.workdir, '.txt')
+    context.check_event(InotifyFileCreatedEvent, newfile)
+    newdir = context.create_random_folder(context.workdir)
+    context.check_event(InotifyDirCreatedEvent, newdir)
+    assert context.check_any()
+
+def test_modify(context):
+    with open(context.subfile, 'wb', buffering=0) as ofile:
+        ofile.write(b'text')
+    context.check_event(InotifyFileModifiedEvent, context.subfile)
+    while True:
+        try:
+            context.observer.modify_queue.get(timeout=0.1)
+            context.missing_events += 1
+        except Empty:
+            break
+    rename(context.subfile, context.subfile+'_new')
+    context.check_event(InotifyDirModifiedEvent, context.subdir)
+    assert context.check_any()
+
+def test_move(context):
+    rename(context.subdir, context.subdir+'_new')
+    context.check_event(InotifyDirMovedEvent, context.subdir)
+    context.check_event(InotifyFileMovedEvent, context.subfile)
+    assert context.check_any()
+
+def test_delete(context):
+    rmtree(context.subdir)
+    context.check_event(InotifyFileDeletedEvent, context.subfile)
+    context.check_event(InotifyDirDeletedEvent, context.subdir)
+    assert context.check_any()
+
+def test_paths(context):
+    context.observer.paths = context.subdir
+    newdir = context.create_random_folder(context.workdir)
+    newfile = context.create_random_file(context.subdir, '.txt')
+    context.check_event(InotifyDirModifiedEvent, context.subdir)
+    context.check_event(InotifyFileCreatedEvent, newfile)
+    context.observer.paths = [newdir, newfile]
+    remove(newfile)
+    context.check_event(InotifyFileDeletedEvent, newfile)
+    assert context.check_any()
+    context.observer.paths = context.workdir
+
+def test_patterns(context):
+    context.observer.patterns = ['*.txt']
+    context.create_random_file(context.subdir, '.bin')
+    newfile = context.create_random_file(context.subdir, '.txt')
+    context.check_event(InotifyFileCreatedEvent, newfile)
+    context.check_empty(InotifyFileCreatedEvent)
+    assert context.check_any()
+    context.observer.patterns = None
+
+def test_exclude(context):
+    context.observer.exclude = ['*.txt']
+    context.create_random_file(context.subdir, '.txt')
+    newfile = context.create_random_file(context.subdir, '.bin')
+    context.check_event(InotifyFileCreatedEvent, newfile)
+    context.check_empty(InotifyFileCreatedEvent)
+    assert context.check_any()
+    context.observer.exclude = None
+
+def test_stress(context):
+    newfile = []
+    for i in range(1024):
+        newfile.append(context.create_random_file(context.subdir, '.txt'))
+    for i in range(1024):
+        context.check_event(InotifyFileCreatedEvent, newfile[i])
+    assert context.check_any()