From ffe512776aa9f63cd421143ab9210dfb3017ca33 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 27 Jun 2019 14:12:01 +0200 Subject: [PATCH] Make files observable and refactor event handlers --- lib/tfw/builtins/ide_event_handler.py | 43 ++++++++++----- .../builtins/log_monitoring_event_handler.py | 40 ++++++-------- lib/tfw/builtins/terminal_event_handler.py | 11 ++-- lib/tfw/components/history_monitor.py | 49 +++++++++++------ lib/tfw/components/inotify/inotify.py | 53 ++++++++++++------- lib/tfw/components/inotify/test_inotify.py | 8 +-- 6 files changed, 123 insertions(+), 81 deletions(-) diff --git a/lib/tfw/builtins/ide_event_handler.py b/lib/tfw/builtins/ide_event_handler.py index 2338a8e..f4025b0 100644 --- a/lib/tfw/builtins/ide_event_handler.py +++ b/lib/tfw/builtins/ide_event_handler.py @@ -10,18 +10,28 @@ from tfw.components.inotify import InotifyObserver LOG = logging.getLogger(__name__) - -class IdeInotifyObserver(InotifyObserver): - def __init__(self, paths): - self.uplink = TFWServerUplinkConnector() - super().__init__(paths) - - def on_modified(self, event): - LOG.debug(event) - self.uplink.send_message({ - 'key': 'ide', - 'data': {'command': 'reload'} - }, Scope.WEBSOCKET) +BUILD_ARTIFACTS = [ + "*.a", + "*.class", + "*.dll", + "*.dylib", + "*.elf", + "*.exe", + "*.jar", + "*.ko", + "*.la", + "*.lib", + "*.lo", + "*.o", + "*.obj", + "*.out", + "*.py[cod]", + "*.so", + "*.so.*", + "*.tar.gz", + "*.zip", + "*__pycache__*" +] class IdeEventHandler(FrontendEventHandlerBase): @@ -60,7 +70,14 @@ class IdeEventHandler(FrontendEventHandlerBase): f'No file(s) in IdeEventHandler working_directory "{directory}"!' ) - self.monitor = IdeInotifyObserver(self.filemanager.allowed_directories) + self.monitor = InotifyObserver(self.filemanager.allowed_directories, exclude=BUILD_ARTIFACTS) + def on_modified(event): + LOG.debug(event) + self.server_connector.send_message({ + 'key': 'ide', + 'data': {'command': 'reload'} + }, Scope.WEBSOCKET) + self.monitor.on_modified = on_modified self.monitor.start() self.commands = { diff --git a/lib/tfw/builtins/log_monitoring_event_handler.py b/lib/tfw/builtins/log_monitoring_event_handler.py index 42a4f79..f1d2919 100644 --- a/lib/tfw/builtins/log_monitoring_event_handler.py +++ b/lib/tfw/builtins/log_monitoring_event_handler.py @@ -13,32 +13,26 @@ LOG = logging.getLogger(__name__) class LogInotifyObserver(InotifyObserver, SupervisorLogMixin): - def __init__(self, process_name, log_tail=0): + def __init__(self, server_connector, process_name, log_tail=0): self.prevent_log_recursion() - self.uplink = TFWServerUplinkConnector() - self.process_name = process_name + self.server_connector = server_connector + self._process_name = process_name self.log_tail = log_tail self.procinfo = self.supervisor.getProcessInfo(self.process_name) - super().__init__( - [dirname(self.procinfo['stdout_logfile']), dirname(self.procinfo['stderr_logfile'])], - [self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']] - ) + InotifyObserver.__init__(self, [self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']]) - def reset(self, process_name, log_tail): - self.process_name = process_name - self.log_tail = log_tail - self.procinfo = self.supervisor.getProcessInfo(self.process_name) - self.paths = [ - dirname(self.procinfo['stdout_logfile']), - dirname(self.procinfo['stderr_logfile']) - ] - self.patterns = [ - self.procinfo['stdout_logfile'], - self.procinfo['stderr_logfile'] - ] + @property + def process_name(self): + return self._process_name + + @process_name.setter + def process_name(self, process_name): + self._process_name = process_name + self.procinfo = self.supervisor.getProcessInfo(process_name) + self.paths = [self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']] def on_modified(self, event): - self.uplink.send_message({ + self.server_connector.send_message({ 'key': 'processlog', 'data': { 'command': 'new_log', @@ -67,7 +61,7 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase): super().__init__(key) self.process_name = process_name self.log_tail = log_tail - self.monitor = LogInotifyObserver(process_name, log_tail) + self.monitor = LogInotifyObserver(self.server_connector, process_name, log_tail) self.monitor.start() self.command_handlers = { @@ -89,7 +83,7 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase): :param data: TFW message data containing 'value' (name of the process to monitor) """ - self.monitor.reset(data['value'], self.log_tail) + self.monitor.process_name = data['value'] def handle_log_tail(self, data): """ @@ -100,7 +94,7 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase): :param data: TFW message data containing 'value' (new tail length) """ - self.monitor.reset(self.process_name, data['value']) + self.monitor.log_tail = data['value'] def cleanup(self): self.monitor.stop() diff --git a/lib/tfw/builtins/terminal_event_handler.py b/lib/tfw/builtins/terminal_event_handler.py index c7104e0..6d2a14a 100644 --- a/lib/tfw/builtins/terminal_event_handler.py +++ b/lib/tfw/builtins/terminal_event_handler.py @@ -4,6 +4,7 @@ import logging from tfw.event_handlers import FrontendEventHandlerBase +from tfw.components import BashMonitor from tfw.components.terminado_mini_server import TerminadoMiniServer from tfw.config import TFWENV from tao.config import TAOENV @@ -21,13 +22,13 @@ class TerminalEventHandler(FrontendEventHandlerBase): a command to be executed. The API of each command is documented in their respective handler. """ - def __init__(self, key, monitor): + def __init__(self, key): """ :param key: key this EventHandler listens to :param monitor: tfw.components.HistoryMonitor instance to read command history from """ super().__init__(key) - self._historymonitor = monitor + self._historymonitor = BashMonitor(self.server_connector, TFWENV.HISTFILE) bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash'] self.terminado_server = TerminadoMiniServer( @@ -42,8 +43,7 @@ class TerminalEventHandler(FrontendEventHandlerBase): 'read': self.read } - if self._historymonitor: - self._historymonitor.start() + self._historymonitor.start() self.terminado_server.listen() @property @@ -84,5 +84,4 @@ class TerminalEventHandler(FrontendEventHandlerBase): def cleanup(self): self.terminado_server.stop() - if self.historymonitor: - self.historymonitor.stop() + self.historymonitor.stop() diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index a11583a..703a68f 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -1,6 +1,10 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +from pwd import getpwnam +from grp import getgrnam +from pathlib import Path +from os import chown from os.path import dirname from re import findall from re import compile as compileregex @@ -22,30 +26,39 @@ class HistoryMonitor(ABC, InotifyObserver): command pattern property and optionally the sanitize_command method. See examples below. """ - def __init__(self, domain, histfile): - self.domain = domain + def __init__(self, uplink, histfile): + self._domain = '' self.histfile = histfile - self._history = [] - self._last_length = len(self._history) - self.uplink = TFWServerUplinkConnector() - super().__init__(dirname(self.histfile), [self.histfile]) + self.history = [] + self._last_length = len(self.history) + self.uplink = uplink + uid = getpwnam('user').pw_uid + gid = getgrnam('users').gr_gid + path = Path(self.histfile) + path.touch() + chown(self.histfile, uid, gid) + super().__init__(self.histfile) + + @property + def domain(self): + return self._domain + + @domain.setter + def domain(self, domain): + self._domain = domain def on_modified(self, event): self._fetch_history() - if self._last_length < len(self._history): - for command in self._history[self._last_length:]: + if self._last_length < len(self.history): + for command in self.history[self._last_length:]: self.send_message(command) - @property - def history(self): - return self._history - def _fetch_history(self): - self._last_length = len(self._history) + self._last_length = len(self.history) with open(self.histfile, 'r') as ifile: pattern = compileregex(self.command_pattern) data = ifile.read() - self._history = [ + self.history = [ self.sanitize_command(command) for command in findall(pattern, data) ] @@ -76,8 +89,9 @@ class BashMonitor(HistoryMonitor): shopt -s histappend unset HISTCONTROL """ - def __init__(self, histfile): - super().__init__('bash', histfile) + def __init__(self, uplink, histfile): + super().__init__(uplink, histfile) + self.domain = 'bash' @property def command_pattern(self): @@ -93,7 +107,8 @@ class GDBMonitor(HistoryMonitor): For this to work "set trace-commands on" must be set in GDB. """ def __init__(self, histfile): - super().__init__('gdb', histfile) + super().__init__(histfile) + self.domain = 'gdb' @property def command_pattern(self): diff --git a/lib/tfw/components/inotify/inotify.py b/lib/tfw/components/inotify/inotify.py index 80b7902..3bbefee 100644 --- a/lib/tfw/components/inotify/inotify.py +++ b/lib/tfw/components/inotify/inotify.py @@ -1,7 +1,8 @@ # pylint: disable=too-few-public-methods -from time import time from typing import Iterable +from time import time +from os.path import abspath, dirname, isfile from watchdog.observers import Observer from watchdog.events import FileSystemMovedEvent, PatternMatchingEventHandler @@ -65,28 +66,44 @@ class InotifyDirDeletedEvent(InotifyEvent): class InotifyObserver: - def __init__(self, paths, patterns=None, exclude=None, recursive=False): - self._paths = paths + def __init__(self, path, patterns=[], exclude=None, recursive=False): + self._files = [] + self._paths = path self._patterns = patterns self._exclude = exclude self._recursive = recursive self._observer = Observer() - self._reset(paths, patterns, exclude) + self._reset() - def _reset(self, paths, patterns, exclude): - _dispatch_event = self._dispatch_event + def _reset(self): + dispatch_event = self._dispatch_event class TransformerEventHandler(PatternMatchingEventHandler): def on_any_event(self, event): - _dispatch_event(event) - self.handler = TransformerEventHandler(patterns, exclude) - self._observer.unschedule_all() - if isinstance(paths, str): - self._observer.schedule(self.handler, paths, self._recursive) - elif isinstance(paths, Iterable): - for path in paths: - self._observer.schedule(self.handler, path, self._recursive) + dispatch_event(event) + + if isinstance(self._paths, str): + self._paths = [self._paths] + if isinstance(self._paths, Iterable): + self._extract_files_from_paths() else: - raise ValueError('Expected one or more strings representing paths.') + raise ValueError('Expected one or more string paths.') + + patterns = self._files+self.patterns + self.handler = TransformerEventHandler(patterns if patterns else None, self.exclude) + self._observer.unschedule_all() + for path in self.paths: + self._observer.schedule(self.handler, path, self._recursive) + + def _extract_files_from_paths(self): + files, paths = [], [] + for path in self._paths: + if isfile(path): + new_file = abspath(path) + files.append(new_file) + paths.append(dirname(new_file)) + else: + paths.append(path) + self._files, self._paths = files, paths @property def paths(self): @@ -95,7 +112,7 @@ class InotifyObserver: @paths.setter def paths(self, paths): self._paths = paths - self._reset(paths, self._patterns, self._exclude) + self._reset() @property def patterns(self): @@ -104,7 +121,7 @@ class InotifyObserver: @patterns.setter def patterns(self, patterns): self._patterns = patterns - self._reset(self._paths, patterns, self._exclude) + self._reset() @property def exclude(self): @@ -113,7 +130,7 @@ class InotifyObserver: @exclude.setter def exclude(self, exclude): self._exclude = exclude - self._reset(self._paths, self._patterns, exclude) + self._reset() def start(self): self._observer.start() diff --git a/lib/tfw/components/inotify/test_inotify.py b/lib/tfw/components/inotify/test_inotify.py index d0ac637..c6d0e12 100644 --- a/lib/tfw/components/inotify/test_inotify.py +++ b/lib/tfw/components/inotify/test_inotify.py @@ -11,8 +11,8 @@ from tempfile import TemporaryDirectory import watchdog import pytest -from inotify import InotifyObserver -from inotify import ( +from .inotify import InotifyObserver +from .inotify import ( InotifyFileCreatedEvent, InotifyFileModifiedEvent, InotifyFileMovedEvent, InotifyFileDeletedEvent, InotifyDirCreatedEvent, InotifyDirModifiedEvent, InotifyDirMovedEvent, InotifyDirDeletedEvent @@ -113,8 +113,8 @@ def test_create(context): assert context.check_any() def test_modify(context): - with open(context.subfile, 'w') as ofile: - ofile.write('text') + with open(context.subfile, 'wb', buffering=0) as ofile: + ofile.write(b'text') context.check_event(InotifyFileModifiedEvent, context.subfile) while True: try: