From 494761d2a76ada2f5265df653944acb80651660c Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 24 Jun 2019 14:29:31 +0200 Subject: [PATCH] Replace mixins with inotify based observers --- lib/tfw/builtins/ide_event_handler.py | 35 ++++---- .../builtins/log_monitoring_event_handler.py | 64 ++++++++++++--- .../process_managing_event_handler.py | 5 +- lib/tfw/components/directory_monitor.py | 81 ------------------- lib/tfw/components/log_monitor.py | 58 ------------- lib/tfw/mixins/monitor_manager_mixin.py | 30 ------- lib/tfw/mixins/observer_mixin.py | 20 ----- 7 files changed, 71 insertions(+), 222 deletions(-) delete mode 100644 lib/tfw/components/directory_monitor.py delete mode 100644 lib/tfw/components/log_monitor.py delete mode 100644 lib/tfw/mixins/monitor_manager_mixin.py delete mode 100644 lib/tfw/mixins/observer_mixin.py diff --git a/lib/tfw/builtins/ide_event_handler.py b/lib/tfw/builtins/ide_event_handler.py index e91213b..2338a8e 100644 --- a/lib/tfw/builtins/ide_event_handler.py +++ b/lib/tfw/builtins/ide_event_handler.py @@ -2,20 +2,29 @@ # All Rights Reserved. See LICENSE file for details. import logging -from os.path import isfile, join, relpath, exists, isdir, realpath -from glob import glob -from fnmatch import fnmatchcase -from typing import Iterable -from tfw.event_handlers import FrontendEventHandlerBase -from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin +from tfw.networking import Scope +from tfw.event_handlers import FrontendEventHandlerBase, TFWServerUplinkConnector from tfw.components import FileManager -from tfw.components.directory_monitor import DirectoryMonitor +from tfw.components.inotify import InotifyObserver LOG = logging.getLogger(__name__) -class IdeEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): +class IdeInotifyObserver(InotifyObserver): + def __init__(self, paths): + self.uplink = TFWServerUplinkConnector() + super().__init__(paths) + + def on_modified(self, event): + LOG.debug(event) + self.uplink.send_message({ + 'key': 'ide', + 'data': {'command': 'reload'} + }, Scope.WEBSOCKET) + + +class IdeEventHandler(FrontendEventHandlerBase): # pylint: disable=too-many-arguments,anomalous-backslash-in-string """ Event handler implementing the backend of our browser based IDE. @@ -51,12 +60,8 @@ class IdeEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): f'No file(s) in IdeEventHandler working_directory "{directory}"!' ) - MonitorManagerMixin.__init__( - self, - DirectoryMonitor, - self.key, - self.filemanager.allowed_directories - ) + self.monitor = IdeInotifyObserver(self.filemanager.allowed_directories) + self.monitor.start() self.commands = { 'read': self.read, @@ -91,7 +96,6 @@ class IdeEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): (new file content) """ - self.monitor.ignore = self.monitor.ignore + 1 try: self.filemanager.file_contents = data['content'] except Exception: # pylint: disable=broad-except @@ -123,7 +127,6 @@ class IdeEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): """ try: self.filemanager.workdir = data['directory'] - self.reload_monitor() try: self.filemanager.filename = self.filemanager.files[0] self.read(data) diff --git a/lib/tfw/builtins/log_monitoring_event_handler.py b/lib/tfw/builtins/log_monitoring_event_handler.py index e324786..42a4f79 100644 --- a/lib/tfw/builtins/log_monitoring_event_handler.py +++ b/lib/tfw/builtins/log_monitoring_event_handler.py @@ -3,14 +3,57 @@ import logging -from tfw.event_handlers import FrontendEventHandlerBase -from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin -from tfw.components.log_monitor import LogMonitor +from os.path import dirname +from tfw.networking import Scope +from tfw.event_handlers import FrontendEventHandlerBase, TFWServerUplinkConnector +from tfw.components.inotify import InotifyObserver +from tfw.mixins.supervisor_mixin import SupervisorLogMixin LOG = logging.getLogger(__name__) -class LogMonitoringEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): +class LogInotifyObserver(InotifyObserver, SupervisorLogMixin): + def __init__(self, process_name, log_tail=0): + self.prevent_log_recursion() + self.uplink = TFWServerUplinkConnector() + self.process_name = process_name + self.log_tail = log_tail + self.procinfo = self.supervisor.getProcessInfo(self.process_name) + super().__init__( + [dirname(self.procinfo['stdout_logfile']), dirname(self.procinfo['stderr_logfile'])], + [self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']] + ) + + def reset(self, process_name, log_tail): + self.process_name = process_name + self.log_tail = log_tail + self.procinfo = self.supervisor.getProcessInfo(self.process_name) + self.paths = [ + dirname(self.procinfo['stdout_logfile']), + dirname(self.procinfo['stderr_logfile']) + ] + self.patterns = [ + self.procinfo['stdout_logfile'], + self.procinfo['stderr_logfile'] + ] + + def on_modified(self, event): + self.uplink.send_message({ + 'key': 'processlog', + 'data': { + 'command': 'new_log', + 'stdout': self.read_stdout(self.process_name, tail=self.log_tail), + 'stderr': self.read_stderr(self.process_name, tail=self.log_tail) + } + }, Scope.BROADCAST) + + @staticmethod + def prevent_log_recursion(): + # This is done to prevent inotify event logs triggering themselves (infinite log recursion) + logging.getLogger('watchdog.observers.inotify_buffer').propagate = False + + +class LogMonitoringEventHandler(FrontendEventHandlerBase): """ Monitors the output of a supervisor process (stdout, stderr) and sends the results to the frontend. @@ -24,12 +67,8 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): super().__init__(key) self.process_name = process_name self.log_tail = log_tail - MonitorManagerMixin.__init__( - self, - LogMonitor, - self.process_name, - self.log_tail - ) + self.monitor = LogInotifyObserver(process_name, log_tail) + self.monitor.start() self.command_handlers = { 'process_name': self.handle_process_name, @@ -40,7 +79,6 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): try: data = message['data'] self.command_handlers[data['command']](data) - self.reload_monitor() except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) @@ -51,7 +89,7 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): :param data: TFW message data containing 'value' (name of the process to monitor) """ - self.set_monitor_args(data['value'], self.log_tail) + self.monitor.reset(data['value'], self.log_tail) def handle_log_tail(self, data): """ @@ -62,7 +100,7 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): :param data: TFW message data containing 'value' (new tail length) """ - self.set_monitor_args(self.process_name, data['value']) + self.monitor.reset(self.process_name, data['value']) def cleanup(self): self.monitor.stop() diff --git a/lib/tfw/builtins/process_managing_event_handler.py b/lib/tfw/builtins/process_managing_event_handler.py index 37141d0..4c7daaa 100644 --- a/lib/tfw/builtins/process_managing_event_handler.py +++ b/lib/tfw/builtins/process_managing_event_handler.py @@ -6,7 +6,6 @@ from xmlrpc.client import Fault as SupervisorFault from tfw.event_handlers import FrontendEventHandlerBase from tfw.mixins.supervisor_mixin import SupervisorMixin, SupervisorLogMixin -from tfw.components.directory_monitor import with_monitor_paused LOG = logging.getLogger(__name__) @@ -36,13 +35,11 @@ class ProcessManagingEventHandler(FrontendEventHandlerBase): Commands available: start, stop, restart, readlog (the names are as self-documenting as it gets) """ - def __init__(self, key, dirmonitor=None, log_tail=0): + def __init__(self, key, log_tail=0): super().__init__(key) - self.monitor = dirmonitor self.processmanager = ProcessManager() self.log_tail = log_tail - @with_monitor_paused def handle_event(self, message): try: data = message['data'] diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py deleted file mode 100644 index 65dc275..0000000 --- a/lib/tfw/components/directory_monitor.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -import logging -from functools import wraps - -from watchdog.events import FileSystemEventHandler as FileSystemWatchdogEventHandler - -from tfw.networking import Scope -from tfw.event_handlers import TFWServerUplinkConnector -from tfw.decorators.rate_limiter import RateLimiter -from tfw.mixins.observer_mixin import ObserverMixin - -LOG = logging.getLogger(__name__) - - -class DirectoryMonitor(ObserverMixin): - def __init__(self, ide_key, directories): - self.eventhandler = IdeReloadWatchdogEventHandler(ide_key) - for directory in directories: - self.observer.schedule(self.eventhandler, directory, recursive=True) - - self.pause, self.resume = self.eventhandler.pause, self.eventhandler.resume - - @property - def ignore(self): - return self.eventhandler.ignore - - @ignore.setter - def ignore(self, value): - self.eventhandler.ignore = value if value >= 0 else 0 - - @property - def pauser(self): - return DirectoryMonitor.Pauser(self) - - class Pauser: - def __init__(self, directory_monitor): - self.directorymonitor = directory_monitor - def __enter__(self): - self.directorymonitor.pause() - def __exit__(self, exc_type, exc_val, exc_tb): - self.directorymonitor.resume() - - -class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler): - def __init__(self, ide_key): - super().__init__() - self.ide_key = ide_key - self.uplink = TFWServerUplinkConnector() - self._paused = False - self.ignore = 0 - - def pause(self): - self._paused = True - - def resume(self): - self._paused = False - - @RateLimiter(rate_per_second=2) - def on_modified(self, event): - if self._paused: - return - if self.ignore > 0: - self.ignore = self.ignore - 1 - return - LOG.debug(event) - self.uplink.send_message({ - 'key': self.ide_key, - 'data': {'command': 'reload'} - }, Scope.WEBSOCKET) - - -def with_monitor_paused(fun): - @wraps(fun) - def wrapper(self, *args, **kwargs): - if self.monitor: - with self.monitor.pauser: - return fun(self, *args, **kwargs) - return fun(self, *args, **kwargs) - return wrapper diff --git a/lib/tfw/components/log_monitor.py b/lib/tfw/components/log_monitor.py deleted file mode 100644 index 14591e8..0000000 --- a/lib/tfw/components/log_monitor.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -import logging -from os.path import dirname - -from watchdog.events import PatternMatchingEventHandler as PatternMatchingWatchdogEventHandler - -from tfw.networking import Scope -from tfw.event_handlers import TFWServerUplinkConnector -from tfw.decorators.rate_limiter import RateLimiter -from tfw.mixins.observer_mixin import ObserverMixin -from tfw.mixins.supervisor_mixin import SupervisorLogMixin - - -class LogMonitor(ObserverMixin): - def __init__(self, process_name, log_tail=0): - self.prevent_log_recursion() - event_handler = SendLogWatchdogEventHandler( - process_name, - log_tail=log_tail - ) - self.observer.schedule( - event_handler, - event_handler.path - ) - - @staticmethod - def prevent_log_recursion(): - # This is done to prevent inotify event logs triggering themselves (infinite log recursion) - logging.getLogger('watchdog.observers.inotify_buffer').propagate = False - - -class SendLogWatchdogEventHandler(PatternMatchingWatchdogEventHandler, SupervisorLogMixin): - def __init__(self, process_name, log_tail=0): - self.process_name = process_name - self.procinfo = self.supervisor.getProcessInfo(self.process_name) - super().__init__([ - self.procinfo['stdout_logfile'], - self.procinfo['stderr_logfile'] - ]) - self.uplink = TFWServerUplinkConnector() - self.log_tail = log_tail - - @property - def path(self): - return dirname(self.procinfo['stdout_logfile']) - - @RateLimiter(rate_per_second=5) - def on_modified(self, event): - self.uplink.send_message({ - 'key': 'processlog', - 'data': { - 'command': 'new_log', - 'stdout': self.read_stdout(self.process_name, tail=self.log_tail), - 'stderr': self.read_stderr(self.process_name, tail=self.log_tail) - } - }, Scope.BROADCAST) diff --git a/lib/tfw/mixins/monitor_manager_mixin.py b/lib/tfw/mixins/monitor_manager_mixin.py deleted file mode 100644 index c153ff9..0000000 --- a/lib/tfw/mixins/monitor_manager_mixin.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -import logging - -LOG = logging.getLogger(__name__) - - -class MonitorManagerMixin: - def __init__(self, monitor_type, *monitor_args): - self._monitor_type = monitor_type - self._monitor = None - self.monitor_args = monitor_args - self.reload_monitor() - - @property - def monitor(self): - return self._monitor - - def set_monitor_args(self, *monitor_args): - self.monitor_args = monitor_args - - def reload_monitor(self): - if self._monitor: - try: - self._monitor.stop() - except KeyError: - LOG.debug('Working directory was removed – ignoring...') - self._monitor = self._monitor_type(*self.monitor_args) - self._monitor.watch() # This runs on a separate thread diff --git a/lib/tfw/mixins/observer_mixin.py b/lib/tfw/mixins/observer_mixin.py deleted file mode 100644 index d5415e5..0000000 --- a/lib/tfw/mixins/observer_mixin.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -from watchdog.observers import Observer - -from tfw.decorators.lazy_property import lazy_property - - -class ObserverMixin: - @lazy_property - def observer(self): - # pylint: disable=no-self-use - return Observer() - - def watch(self): - self.observer.start() - - def stop(self): - self.observer.stop() - self.observer.join()