From 16eb1ba780cfc54d7d12c84179aad9b41f01d38f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 16 May 2019 23:44:44 +0200 Subject: [PATCH 001/174] Name new release --- .drone.yml | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index d998013..1bbfe8e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -10,4 +10,4 @@ pipeline: - docker push eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG} when: event: 'tag' - branch: refs/tags/ocicat-20* + branch: refs/tags/chausie-20* diff --git a/VERSION b/VERSION index 4ed6a03..63a1a7e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -ocicat +chausie From 743ff8be4e158cad13c2b0b90c7f3c4fc9ae18de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 17 May 2019 00:03:46 +0200 Subject: [PATCH 002/174] Update packages --- requirements.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index c2ab3ed..1b5f83c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -tornado==5.1 -pyzmq==17.1.2 -transitions==0.6.6 -terminado==0.8.1 -watchdog==0.8.3 -PyYAML==3.13 -Jinja2==2.10 -cryptography==2.3.1 -python-dateutil==2.7.3 +tornado>=6.0.0,<7.0.0 +pyzmq>=17.0.0,<18.0.0 +transitions>=0.0.0,<1.0.0 +terminado>=0.0.0,<1.0.0 +watchdog>=0.0.0,<1.0.0 +PyYAML>=5.0.0,<6.0.0 +Jinja2>=2.0.0,<3.0.0 +cryptography>=2.0.0,<3.0.0 +python-dateutil>=2.0.0,<3.0.0 From 2a3d29f0801d3c007d8656611df198b76023b535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 17 May 2019 00:04:36 +0200 Subject: [PATCH 003/174] Remove git hooks --- .git-hooks/apply_hooks.sh | 22 ---------------------- .git-hooks/pre-push.sh | 18 ------------------ 2 files changed, 40 deletions(-) delete mode 100755 .git-hooks/apply_hooks.sh delete mode 100755 .git-hooks/pre-push.sh diff --git a/.git-hooks/apply_hooks.sh b/.git-hooks/apply_hooks.sh deleted file mode 100755 index f95f8f4..0000000 --- a/.git-hooks/apply_hooks.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -eu -set -o pipefail -set -o errtrace -shopt -s expand_aliases - -[ "$(uname)" == "Darwin" ] && alias readlink="greadlink" || : - -GREEN='\033[0;32m' -NC='\033[0m' - - -here="$(dirname "$(readlink -f "$0")")" -cd "${here}/../.git/hooks" - -rm -f pre-push pre-commit || : -prepush_script="../../.git-hooks/pre-push.sh" -precommit_script="../../.git-hooks/pre-commit.sh" -[ -f "${prepush_script}" ] && ln -s "${prepush_script}" pre-push -[ -f "${precommit_script}" ] && ln -s "${precommit_script}" pre-commit - -echo -e "\n${GREEN}Done! Hooks applied, you can start committing and pushing!${NC}\n" diff --git a/.git-hooks/pre-push.sh b/.git-hooks/pre-push.sh deleted file mode 100755 index e2025ac..0000000 --- a/.git-hooks/pre-push.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -eu -set -o pipefail -set -o errtrace -shopt -s expand_aliases - -RED='\033[0;31m' -GREEN='\033[0;32m' -NC='\033[0m' - - -echo -e "Running pylint...\n" -if pylint lib; then - echo -e "\n${GREEN}Pylint found no errors!${NC}\n" -else - echo -e "\n${RED}Pylint failed with errors${NC}\n" - exit 1 -fi From 7d50ee607afb00662993cd26f5ed5c63ccfa4705 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 20 May 2019 11:06:57 +0200 Subject: [PATCH 004/174] Close ZMQ sockets gracefully --- lib/tfw/event_handler_base/event_handler_base.py | 4 ++++ .../networking/event_handlers/server_connector.py | 12 +++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/tfw/event_handler_base/event_handler_base.py b/lib/tfw/event_handler_base/event_handler_base.py index c013a8d..18541ee 100644 --- a/lib/tfw/event_handler_base/event_handler_base.py +++ b/lib/tfw/event_handler_base/event_handler_base.py @@ -106,6 +106,10 @@ class EventHandlerBase(ABC): self.server_connector.unsubscribe(key) self.keys.remove(key) + def stop(self): + self.server_connector.close() + self.cleanup() + def cleanup(self): """ Perform cleanup actions such as releasing database diff --git a/lib/tfw/networking/event_handlers/server_connector.py b/lib/tfw/networking/event_handlers/server_connector.py index 7d24803..4c2a2b9 100644 --- a/lib/tfw/networking/event_handlers/server_connector.py +++ b/lib/tfw/networking/event_handlers/server_connector.py @@ -19,6 +19,7 @@ class ServerDownlinkConnector(ZMQConnectorBase): super(ServerDownlinkConnector, self).__init__(zmq_context) self._zmq_sub_socket = self._zmq_context.socket(zmq.SUB) self._zmq_sub_socket.connect(f'tcp://localhost:{TFWENV.PUBLISHER_PORT}') + self._zmq_sub_socket.setsockopt(zmq.RCVHWM,0) self._zmq_sub_stream = ZMQStream(self._zmq_sub_socket) self.subscribe = partial(self._zmq_sub_socket.setsockopt_string, zmq.SUBSCRIBE) @@ -28,6 +29,9 @@ class ServerDownlinkConnector(ZMQConnectorBase): callback = with_deserialize_tfw_msg(callback) self._zmq_sub_stream.on_recv(callback) + def close(self): + self._zmq_sub_stream.close() + class ServerUplinkConnector(ZMQConnectorBase): """ @@ -37,6 +41,7 @@ class ServerUplinkConnector(ZMQConnectorBase): super(ServerUplinkConnector, self).__init__(zmq_context) self._zmq_push_socket = self._zmq_context.socket(zmq.PUSH) self._zmq_push_socket.connect(f'tcp://localhost:{TFWENV.RECEIVER_PORT}') + self._zmq_push_socket.setsockopt(zmq.SNDHWM,0) def send_to_eventhandler(self, message): """ @@ -74,6 +79,11 @@ class ServerUplinkConnector(ZMQConnectorBase): 'data': message }) + def close(self): + self._zmq_push_socket.close() + class ServerConnector(ServerUplinkConnector, ServerDownlinkConnector): - pass + def close(self): + ServerUplinkConnector.close(self) + ServerDownlinkConnector.close(self) From 71f05fe92ccb9b3912dbd779393622184551810f Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 31 May 2019 11:13:33 +0200 Subject: [PATCH 005/174] Fix terminal writing error on challenge startup --- lib/tfw/components/terminado_mini_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/tfw/components/terminado_mini_server.py b/lib/tfw/components/terminado_mini_server.py index 3726097..e196c1f 100644 --- a/lib/tfw/components/terminado_mini_server.py +++ b/lib/tfw/components/terminado_mini_server.py @@ -30,6 +30,8 @@ class TerminadoMiniServer: @property def pty(self): + if self.term_manager.terminal is None: + self.term_manager.get_terminal() return self.term_manager.terminal.ptyproc class ResetterTermSocket(TermSocket): # pylint: disable=abstract-method From de6afdc84a51cca1bb569618b2fde9f7c9114243 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 31 May 2019 13:10:33 +0200 Subject: [PATCH 006/174] Make Terminado stoppable --- lib/tfw/components/terminado_mini_server.py | 3 +++ lib/tfw/components/terminal_event_handler.py | 1 + 2 files changed, 4 insertions(+) diff --git a/lib/tfw/components/terminado_mini_server.py b/lib/tfw/components/terminado_mini_server.py index e196c1f..0f77c34 100644 --- a/lib/tfw/components/terminado_mini_server.py +++ b/lib/tfw/components/terminado_mini_server.py @@ -45,6 +45,9 @@ class TerminadoMiniServer: def listen(self): self.application.listen(self.port) + def stop(self): + self.term_manager.shutdown() + if __name__ == '__main__': LOG.info('Terminado Mini Server listening on %s', TFWENV.TERMINADO_PORT) diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/components/terminal_event_handler.py index 7fbd74b..e49415c 100644 --- a/lib/tfw/components/terminal_event_handler.py +++ b/lib/tfw/components/terminal_event_handler.py @@ -83,5 +83,6 @@ class TerminalEventHandler(EventHandlerBase): return data def cleanup(self): + self.terminado_server.stop() if self.historymonitor: self.historymonitor.stop() From 26c6c5d1e657c476775a5886baeec1d5ca5efebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 21 May 2019 13:44:02 +0200 Subject: [PATCH 007/174] Refactor EventHandlerConnector family of classes --- .../networking/server/event_handler_connector.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/tfw/networking/server/event_handler_connector.py b/lib/tfw/networking/server/event_handler_connector.py index 5075c56..bc6c9be 100644 --- a/lib/tfw/networking/server/event_handler_connector.py +++ b/lib/tfw/networking/server/event_handler_connector.py @@ -21,6 +21,10 @@ class EventHandlerDownlinkConnector(ZMQConnectorBase): self._zmq_pull_socket.bind(address) LOG.debug('Pull socket bound to %s', address) + def register_callback(self, callback): + callback = with_deserialize_tfw_msg(callback) + self._zmq_pull_stream.on_recv(callback) + class EventHandlerUplinkConnector(ZMQConnectorBase): def __init__(self, zmq_context=None): @@ -30,11 +34,9 @@ class EventHandlerUplinkConnector(ZMQConnectorBase): self._zmq_pub_socket.bind(address) LOG.debug('Pub socket bound to %s', address) - -class EventHandlerConnector(EventHandlerDownlinkConnector, EventHandlerUplinkConnector): - def register_callback(self, callback): - callback = with_deserialize_tfw_msg(callback) - self._zmq_pull_stream.on_recv(callback) - def send_message(self, message: dict): self._zmq_pub_socket.send_multipart(serialize_tfw_msg(message)) + + +class EventHandlerConnector(EventHandlerDownlinkConnector, EventHandlerUplinkConnector): + pass From 6b23b863ed47b7bf52b89fdbaba0aa7b8c5aa0dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 21 May 2019 13:55:28 +0200 Subject: [PATCH 008/174] Fix code formatting --- lib/tfw/networking/event_handlers/server_connector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tfw/networking/event_handlers/server_connector.py b/lib/tfw/networking/event_handlers/server_connector.py index 4c2a2b9..ab9973b 100644 --- a/lib/tfw/networking/event_handlers/server_connector.py +++ b/lib/tfw/networking/event_handlers/server_connector.py @@ -19,7 +19,7 @@ class ServerDownlinkConnector(ZMQConnectorBase): super(ServerDownlinkConnector, self).__init__(zmq_context) self._zmq_sub_socket = self._zmq_context.socket(zmq.SUB) self._zmq_sub_socket.connect(f'tcp://localhost:{TFWENV.PUBLISHER_PORT}') - self._zmq_sub_socket.setsockopt(zmq.RCVHWM,0) + self._zmq_sub_socket.setsockopt(zmq.RCVHWM, 0) self._zmq_sub_stream = ZMQStream(self._zmq_sub_socket) self.subscribe = partial(self._zmq_sub_socket.setsockopt_string, zmq.SUBSCRIBE) @@ -41,7 +41,7 @@ class ServerUplinkConnector(ZMQConnectorBase): super(ServerUplinkConnector, self).__init__(zmq_context) self._zmq_push_socket = self._zmq_context.socket(zmq.PUSH) self._zmq_push_socket.connect(f'tcp://localhost:{TFWENV.RECEIVER_PORT}') - self._zmq_push_socket.setsockopt(zmq.SNDHWM,0) + self._zmq_push_socket.setsockopt(zmq.SNDHWM, 0) def send_to_eventhandler(self, message): """ From 6431fac9b15cb5ac3619d83d0d165bbf509c2a34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 21 May 2019 13:57:56 +0200 Subject: [PATCH 009/174] Set ZMQ HWM in EventHandlerConnector to infinite --- lib/tfw/networking/server/event_handler_connector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/tfw/networking/server/event_handler_connector.py b/lib/tfw/networking/server/event_handler_connector.py index bc6c9be..8b13338 100644 --- a/lib/tfw/networking/server/event_handler_connector.py +++ b/lib/tfw/networking/server/event_handler_connector.py @@ -16,6 +16,7 @@ class EventHandlerDownlinkConnector(ZMQConnectorBase): def __init__(self, zmq_context=None): super(EventHandlerDownlinkConnector, self).__init__(zmq_context) self._zmq_pull_socket = self._zmq_context.socket(zmq.PULL) + self._zmq_pull_socket.setsockopt(zmq.RCVHWM, 0) self._zmq_pull_stream = ZMQStream(self._zmq_pull_socket) address = f'tcp://*:{TFWENV.RECEIVER_PORT}' self._zmq_pull_socket.bind(address) @@ -30,6 +31,7 @@ class EventHandlerUplinkConnector(ZMQConnectorBase): def __init__(self, zmq_context=None): super(EventHandlerUplinkConnector, self).__init__(zmq_context) self._zmq_pub_socket = self._zmq_context.socket(zmq.PUB) + self._zmq_pub_socket.setsockopt(zmq.SNDHWM, 0) address = f'tcp://*:{TFWENV.PUBLISHER_PORT}' self._zmq_pub_socket.bind(address) LOG.debug('Pub socket bound to %s', address) From 613919a5b60d4d2876c20c27e69647fbe197db22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 21 May 2019 13:59:42 +0200 Subject: [PATCH 010/174] Implement closing EventHandlerConnector --- lib/tfw/networking/server/event_handler_connector.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/tfw/networking/server/event_handler_connector.py b/lib/tfw/networking/server/event_handler_connector.py index 8b13338..7b56a88 100644 --- a/lib/tfw/networking/server/event_handler_connector.py +++ b/lib/tfw/networking/server/event_handler_connector.py @@ -26,6 +26,9 @@ class EventHandlerDownlinkConnector(ZMQConnectorBase): callback = with_deserialize_tfw_msg(callback) self._zmq_pull_stream.on_recv(callback) + def close(self): + self._zmq_pull_stream.close() + class EventHandlerUplinkConnector(ZMQConnectorBase): def __init__(self, zmq_context=None): @@ -39,6 +42,11 @@ class EventHandlerUplinkConnector(ZMQConnectorBase): def send_message(self, message: dict): self._zmq_pub_socket.send_multipart(serialize_tfw_msg(message)) + def close(self): + self._zmq_pub_socket.close() + class EventHandlerConnector(EventHandlerDownlinkConnector, EventHandlerUplinkConnector): - pass + def close(self): + EventHandlerDownlinkConnector.close(self) + EventHandlerUplinkConnector.close(self) From 01d90035018a2132e4896ff6438e3f71e4b78003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sun, 26 May 2019 18:26:33 +0200 Subject: [PATCH 011/174] Rework whole TFW networking model --- lib/tfw/components/__init__.py | 1 + lib/tfw/components/directory_monitor.py | 6 +- .../directory_monitoring_event_handler.py | 4 +- .../directory_snapshotting_event_handler.py | 4 +- lib/tfw/components/frontend_event_handler.py | 62 ++++++++ .../components/fsm_managing_event_handler.py | 7 +- lib/tfw/components/ide_event_handler.py | 4 +- lib/tfw/components/log_monitor.py | 6 +- .../log_monitoring_event_handler.py | 4 +- lib/tfw/components/pipe_io_event_handler.py | 4 +- .../process_managing_event_handler.py | 4 +- lib/tfw/components/terminal_event_handler.py | 4 +- lib/tfw/event_handler_base/__init__.py | 2 +- .../boradcasting_event_handler.py | 3 +- .../event_handler_base/event_handler_base.py | 12 +- .../event_handlers/server_connector.py | 46 ++---- lib/tfw/networking/message_sender.py | 32 ++-- lib/tfw/networking/server/tfw_server.py | 88 +---------- .../networking/server/zmq_websocket_proxy.py | 141 ++++-------------- 19 files changed, 155 insertions(+), 279 deletions(-) create mode 100644 lib/tfw/components/frontend_event_handler.py diff --git a/lib/tfw/components/__init__.py b/lib/tfw/components/__init__.py index 7c3f76d..7d2ebc0 100644 --- a/lib/tfw/components/__init__.py +++ b/lib/tfw/components/__init__.py @@ -14,3 +14,4 @@ from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler, P from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler from .commands_equal import CommandsEqual +from .frontend_event_handler import FrontendEventHandler diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index f65c8ff..6aa2ca0 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -5,7 +5,7 @@ from functools import wraps from watchdog.events import FileSystemEventHandler as FileSystemWatchdogEventHandler -from tfw.networking.event_handlers.server_connector import ServerUplinkConnector +from tfw.networking.event_handlers.server_connector import ServerUplinkConnector, Scope from tfw.decorators.rate_limiter import RateLimiter from tfw.mixins.observer_mixin import ObserverMixin @@ -65,10 +65,10 @@ class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler): self.ignore = self.ignore - 1 return LOG.debug(event) - self.uplink.send({ + self.uplink.send_message({ 'key': self.ide_key, 'data': {'command': 'reload'} - }) + }, Scope.WEBSOCKET) def with_monitor_paused(fun): diff --git a/lib/tfw/components/directory_monitoring_event_handler.py b/lib/tfw/components/directory_monitoring_event_handler.py index 8d97022..03c4a3a 100644 --- a/lib/tfw/components/directory_monitoring_event_handler.py +++ b/lib/tfw/components/directory_monitoring_event_handler.py @@ -3,7 +3,7 @@ from os.path import isdir, exists -from tfw.event_handler_base import EventHandlerBase +from tfw.event_handler_base import FrontendEventHandlerBase from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin from tfw.components.directory_monitor import DirectoryMonitor from tfw.config.logs import logging @@ -11,7 +11,7 @@ from tfw.config.logs import logging LOG = logging.getLogger(__name__) -class DirectoryMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): +class DirectoryMonitoringEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): def __init__(self, key, directory): super().__init__(key) self._directory = directory diff --git a/lib/tfw/components/directory_snapshotting_event_handler.py b/lib/tfw/components/directory_snapshotting_event_handler.py index 8b6f384..6a285a8 100644 --- a/lib/tfw/components/directory_snapshotting_event_handler.py +++ b/lib/tfw/components/directory_snapshotting_event_handler.py @@ -8,7 +8,7 @@ from datetime import datetime from dateutil import parser as dateparser -from tfw.event_handler_base import EventHandlerBase +from tfw.event_handler_base import FrontendEventHandlerBase from tfw.components.snapshot_provider import SnapshotProvider from tfw.config import TFWENV from tfw.config.logs import logging @@ -16,7 +16,7 @@ from tfw.config.logs import logging LOG = logging.getLogger(__name__) -class DirectorySnapshottingEventHandler(EventHandlerBase): +class DirectorySnapshottingEventHandler(FrontendEventHandlerBase): def __init__(self, key, directories, exclude_unix_patterns=None): super().__init__(key) self.snapshot_providers = {} diff --git a/lib/tfw/components/frontend_event_handler.py b/lib/tfw/components/frontend_event_handler.py new file mode 100644 index 0000000..898bb7e --- /dev/null +++ b/lib/tfw/components/frontend_event_handler.py @@ -0,0 +1,62 @@ +from abc import ABC, abstractmethod +from contextlib import suppress + +from tfw.networking.message_sender import MessageSender +from tfw.event_handler_base import FrontendEventHandlerBase + + +class FrontendEventHandler(FrontendEventHandlerBase): + def __init__(self): + frontend_keys = ('message', 'queueMessages', 'dashboard', 'console') + self._frontend_message_storage = FrontendMessageStorage(frontend_keys) + super().__init__((*frontend_keys, 'recover')) + + def handle_event(self, message): + self._frontend_message_storage.save_message(message) + if message['key'] == 'recover': + self.recover_frontend() + return message + + def recover_frontend(self): + for message in self._frontend_message_storage.messages: + self.send_message(message) + + +class MessageStorage(ABC): + def __init__(self): + self._messages = [] + + def save_message(self, message): + with suppress(KeyError, AttributeError): + if self._filter_message(message): + self._messages.extend(self._transform_message(message)) + + @abstractmethod + def _filter_message(self, message): + raise NotImplementedError + + def _transform_message(self, message): # pylint: disable=no-self-use + yield message + + def clear(self): + self._messages.clear() + + @property + def messages(self): + yield from self._messages + + +class FrontendMessageStorage(MessageStorage): + def __init__(self, keys): + self._keys = keys + super().__init__() + + def _filter_message(self, message): + key = message['key'] + return key in self._keys + + def _transform_message(self, message): + if message['key'] == 'queueMessages': + yield from MessageSender.generate_messages_from_queue(message) + else: + yield message diff --git a/lib/tfw/components/fsm_managing_event_handler.py b/lib/tfw/components/fsm_managing_event_handler.py index 73ebe10..3069fc1 100644 --- a/lib/tfw/components/fsm_managing_event_handler.py +++ b/lib/tfw/components/fsm_managing_event_handler.py @@ -1,14 +1,15 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw.event_handler_base import EventHandlerBase +from tfw.event_handler_base import FrontendEventHandlerBase from tfw.crypto import KeyManager, sign_message, verify_message from tfw.config.logs import logging +from tfw.networking.event_handlers.server_connector import Scope LOG = logging.getLogger(__name__) -class FSMManagingEventHandler(EventHandlerBase): +class FSMManagingEventHandler(FrontendEventHandlerBase): """ EventHandler responsible for managing the state machine of the framework (TFW FSM). @@ -42,7 +43,7 @@ class FSMManagingEventHandler(EventHandlerBase): fsm_update_message = self._fsm_updater.fsm_update sign_message(self.auth_key, message) sign_message(self.auth_key, fsm_update_message) - self.server_connector.broadcast(fsm_update_message) + self.server_connector.send_message(fsm_update_message, Scope.BROADCAST) return message except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/lib/tfw/components/ide_event_handler.py b/lib/tfw/components/ide_event_handler.py index 7e242e5..534de6a 100644 --- a/lib/tfw/components/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler.py @@ -6,7 +6,7 @@ from glob import glob from fnmatch import fnmatchcase from typing import Iterable -from tfw.event_handler_base import EventHandlerBase +from tfw.event_handler_base import FrontendEventHandlerBase from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin from tfw.components.directory_monitor import DirectoryMonitor from tfw.config.logs import logging @@ -102,7 +102,7 @@ class FileManager: # pylint: disable=too-many-instance-attributes return relpath(self._filepath(filename), start=self._workdir) -class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): +class IdeEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): # pylint: disable=too-many-arguments,anomalous-backslash-in-string """ Event handler implementing the backend of our browser based IDE. diff --git a/lib/tfw/components/log_monitor.py b/lib/tfw/components/log_monitor.py index cf06a0a..2ee6d96 100644 --- a/lib/tfw/components/log_monitor.py +++ b/lib/tfw/components/log_monitor.py @@ -6,7 +6,7 @@ from os.path import dirname from watchdog.events import PatternMatchingEventHandler as PatternMatchingWatchdogEventHandler -from tfw.networking.event_handlers.server_connector import ServerUplinkConnector +from tfw.networking.event_handlers.server_connector import ServerUplinkConnector, Scope from tfw.decorators.rate_limiter import RateLimiter from tfw.mixins.observer_mixin import ObserverMixin from tfw.mixins.supervisor_mixin import SupervisorLogMixin @@ -47,11 +47,11 @@ class SendLogWatchdogEventHandler(PatternMatchingWatchdogEventHandler, Superviso @RateLimiter(rate_per_second=5) def on_modified(self, event): - self.uplink.send({ + self.uplink.send_message({ 'key': 'processlog', 'data': { 'command': 'new_log', 'stdout': self.read_stdout(self.process_name, tail=self.log_tail), 'stderr': self.read_stderr(self.process_name, tail=self.log_tail) } - }) + }, Scope.BROADCAST) diff --git a/lib/tfw/components/log_monitoring_event_handler.py b/lib/tfw/components/log_monitoring_event_handler.py index 0bc7ab2..d384323 100644 --- a/lib/tfw/components/log_monitoring_event_handler.py +++ b/lib/tfw/components/log_monitoring_event_handler.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw.event_handler_base import EventHandlerBase +from tfw.event_handler_base import FrontendEventHandlerBase from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin from tfw.components.log_monitor import LogMonitor from tfw.config.logs import logging @@ -9,7 +9,7 @@ from tfw.config.logs import logging LOG = logging.getLogger(__name__) -class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): +class LogMonitoringEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): """ Monitors the output of a supervisor process (stdout, stderr) and sends the results to the frontend. diff --git a/lib/tfw/components/pipe_io_event_handler.py b/lib/tfw/components/pipe_io_event_handler.py index 0a94e57..37b5b89 100644 --- a/lib/tfw/components/pipe_io_event_handler.py +++ b/lib/tfw/components/pipe_io_event_handler.py @@ -56,7 +56,7 @@ class PipeIOEventHandler(PipeIOEventHandlerBase): def handle_pipe_event(self, message_bytes): json = loads(message_bytes) - self.server_connector.send(json) + self.send_message(json) class TransformerPipeIOEventHandler(PipeIOEventHandlerBase): @@ -93,7 +93,7 @@ class TransformerPipeIOEventHandler(PipeIOEventHandlerBase): transformed_bytes = self._transform_in(message_bytes) if transformed_bytes: json_message = loads(transformed_bytes) - self.server_connector.send(json_message) + self.send_message(json_message) class CommandEventHandler(PipeIOEventHandler): diff --git a/lib/tfw/components/process_managing_event_handler.py b/lib/tfw/components/process_managing_event_handler.py index 61b6c62..a87bdd6 100644 --- a/lib/tfw/components/process_managing_event_handler.py +++ b/lib/tfw/components/process_managing_event_handler.py @@ -3,7 +3,7 @@ from xmlrpc.client import Fault as SupervisorFault -from tfw.event_handler_base import EventHandlerBase +from tfw.event_handler_base import FrontendEventHandlerBase from tfw.mixins.supervisor_mixin import SupervisorMixin, SupervisorLogMixin from tfw.components.directory_monitor import with_monitor_paused from tfw.config.logs import logging @@ -23,7 +23,7 @@ class ProcessManager(SupervisorMixin, SupervisorLogMixin): return self.commands[command](process_name) -class ProcessManagingEventHandler(EventHandlerBase): +class ProcessManagingEventHandler(FrontendEventHandlerBase): """ Event handler that can manage processes managed by supervisor. diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/components/terminal_event_handler.py index e49415c..f45bc8a 100644 --- a/lib/tfw/components/terminal_event_handler.py +++ b/lib/tfw/components/terminal_event_handler.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw.event_handler_base import EventHandlerBase +from tfw.event_handler_base import FrontendEventHandlerBase from tfw.components.terminado_mini_server import TerminadoMiniServer from tfw.config import TFWENV from tfw.config.logs import logging @@ -10,7 +10,7 @@ from tao.config import TAOENV LOG = logging.getLogger(__name__) -class TerminalEventHandler(EventHandlerBase): +class TerminalEventHandler(FrontendEventHandlerBase): """ Event handler responsible for managing terminal sessions for frontend xterm sessions to connect to. You need to instanciate this in order for frontend diff --git a/lib/tfw/event_handler_base/__init__.py b/lib/tfw/event_handler_base/__init__.py index fd50525..65ca6e1 100644 --- a/lib/tfw/event_handler_base/__init__.py +++ b/lib/tfw/event_handler_base/__init__.py @@ -1,6 +1,6 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from .event_handler_base import EventHandlerBase +from .event_handler_base import EventHandlerBase, FrontendEventHandlerBase from .boradcasting_event_handler import BroadcastingEventHandler from .fsm_aware_event_handler import FSMAwareEventHandler diff --git a/lib/tfw/event_handler_base/boradcasting_event_handler.py b/lib/tfw/event_handler_base/boradcasting_event_handler.py index 59ee493..cfda694 100644 --- a/lib/tfw/event_handler_base/boradcasting_event_handler.py +++ b/lib/tfw/event_handler_base/boradcasting_event_handler.py @@ -4,6 +4,7 @@ from abc import ABC from tfw.event_handler_base.event_handler_base import EventHandlerBase +from tfw.networking.event_handlers.server_connector import Scope from tfw.crypto import message_checksum @@ -27,4 +28,4 @@ class BroadcastingEventHandler(EventHandlerBase, ABC): response = self.dispatch_handling(message) if response: self.own_message_hashes.append(message_checksum(response)) - self.server_connector.broadcast(response) + self.server_connector.send_message(response, Scope.BROADCAST) diff --git a/lib/tfw/event_handler_base/event_handler_base.py b/lib/tfw/event_handler_base/event_handler_base.py index 18541ee..f51c668 100644 --- a/lib/tfw/event_handler_base/event_handler_base.py +++ b/lib/tfw/event_handler_base/event_handler_base.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from inspect import currentframe from typing import Iterable -from tfw.networking.event_handlers.server_connector import ServerConnector +from tfw.networking.event_handlers.server_connector import ServerConnector, Scope from tfw.config.logs import logging LOG = logging.getLogger(__name__) @@ -48,7 +48,10 @@ class EventHandlerBase(ABC): response = self.dispatch_handling(message) if response: - self.server_connector.send(response) + self.send_message(response) + + def send_message(self, message): + self.server_connector.send_message(message) def check_key(self, message): """ @@ -128,3 +131,8 @@ class EventHandlerBase(ABC): instance for instance in locals_values if isinstance(instance, cls) } + + +class FrontendEventHandlerBase(EventHandlerBase): # pylint: disable=abstract-method + def send_message(self, message): + self.server_connector.send_message(message, Scope.WEBSOCKET) diff --git a/lib/tfw/networking/event_handlers/server_connector.py b/lib/tfw/networking/event_handlers/server_connector.py index ab9973b..73b04e0 100644 --- a/lib/tfw/networking/event_handlers/server_connector.py +++ b/lib/tfw/networking/event_handlers/server_connector.py @@ -2,6 +2,7 @@ # All Rights Reserved. See LICENSE file for details. from functools import partial +from enum import Enum import zmq from zmq.eventloop.zmqstream import ZMQStream @@ -33,52 +34,23 @@ class ServerDownlinkConnector(ZMQConnectorBase): self._zmq_sub_stream.close() +class Scope(Enum): + ZMQ = 'zmq' + WEBSOCKET = 'websocket' + BROADCAST = 'broadcast' + + class ServerUplinkConnector(ZMQConnectorBase): - """ - Class capable of sending messages to the TFW server and event handlers. - """ def __init__(self, zmq_context=None): super(ServerUplinkConnector, self).__init__(zmq_context) self._zmq_push_socket = self._zmq_context.socket(zmq.PUSH) self._zmq_push_socket.connect(f'tcp://localhost:{TFWENV.RECEIVER_PORT}') self._zmq_push_socket.setsockopt(zmq.SNDHWM, 0) - def send_to_eventhandler(self, message): - """ - Send a message to an event handler through the TFW server. - - This envelopes the desired message in the 'data' field of the message to - TFWServer, which will mirror it to event handlers. - - :param message: JSON message you want to send - """ - self.send({ - 'key': 'mirror', - 'data': message - }) - - def send(self, message): - """ - Send a message to the frontend through the TFW server. - - :param message: JSON message you want to send - """ + def send_message(self, message, scope=Scope.ZMQ): + message['scope'] = scope.value self._zmq_push_socket.send_multipart(serialize_tfw_msg(message)) - def broadcast(self, message): - """ - Broadast a message through the TFW server. - - This envelopes the desired message in the 'data' field of the message to - TFWServer, which will broadast it. - - :param message: JSON message you want to send - """ - self.send({ - 'key': 'broadcast', - 'data': message - }) - def close(self): self._zmq_push_socket.close() diff --git a/lib/tfw/networking/message_sender.py b/lib/tfw/networking/message_sender.py index 58eb1c9..1232aaa 100644 --- a/lib/tfw/networking/message_sender.py +++ b/lib/tfw/networking/message_sender.py @@ -19,14 +19,14 @@ class MessageSender: :param originator: name of sender to be displayed on the frontend :param message: message to send """ - data = { - 'originator': originator, - 'message': message - } - self.server_connector.send({ + message = { 'key': self.key, - 'data': data - }) + 'data': { + 'originator': originator, + 'message': message + } + } + self.server_connector.send_message(message) def queue_messages(self, originator, messages): """ @@ -34,16 +34,16 @@ class MessageSender: :param originator: name of sender to be displayed on the frontend :param messages: list of messages to queue """ - data = { - 'messages': [ - {'message': message, 'originator': originator} - for message in messages - ] - } - self.server_connector.send({ + message = { 'key': self.queue_key, - 'data': data - }) + 'data': { + 'messages': [ + {'message': message, 'originator': originator} + for message in messages + ] + } + } + self.server_connector.send_message(message) @staticmethod def generate_messages_from_queue(queue_message): diff --git a/lib/tfw/networking/server/tfw_server.py b/lib/tfw/networking/server/tfw_server.py index a1a0f5f..ed28e62 100644 --- a/lib/tfw/networking/server/tfw_server.py +++ b/lib/tfw/networking/server/tfw_server.py @@ -1,114 +1,28 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from abc import ABC, abstractmethod -from contextlib import suppress - from tornado.web import Application from tfw.networking.server.zmq_websocket_proxy import ZMQWebSocketProxy -from tfw.networking.event_handlers.server_connector import ServerUplinkConnector from tfw.networking.server.event_handler_connector import EventHandlerConnector -from tfw.networking.message_sender import MessageSender -from tfw.networking.fsm_aware import FSMAware -from tfw.crypto import KeyManager, verify_message, sign_message from tfw.config.logs import logging LOG = logging.getLogger(__name__) -class TFWServer(FSMAware): +class TFWServer: """ This class handles the proxying of messages between the frontend and event handers. It proxies messages from the "/ws" route to all event handlers subscribed to a ZMQ SUB socket. """ def __init__(self): - super().__init__() self._event_handler_connector = EventHandlerConnector() - self._uplink_connector = ServerUplinkConnector() - self._auth_key = KeyManager().auth_key - self.application = Application([( r'/ws', ZMQWebSocketProxy, { 'event_handler_connector': self._event_handler_connector, - 'proxy_filters_and_callbacks': { - 'message_handlers': [ - self.handle_trigger, - self.handle_recover, - self.handle_fsm_update - ], - 'frontend_message_handlers': [self.save_frontend_messages] - } } )]) - self._frontend_messages = FrontendMessageStorage() - - def handle_trigger(self, message): - if 'trigger' in message: - LOG.debug('Executing handler for trigger "%s"', message.get('trigger', '')) - fsm_eh_command = { - 'key': 'fsm', - 'data': { - 'command': 'trigger', - 'value': message['trigger'] - } - } - if verify_message(self._auth_key, message): - sign_message(self._auth_key, fsm_eh_command) - self._uplink_connector.send_to_eventhandler(fsm_eh_command) - - def handle_recover(self, message): - if message['key'] == 'recover': - self._frontend_messages.replay_messages(self._uplink_connector) - self._frontend_messages.clear() - - def handle_fsm_update(self, message): - self.update_fsm_data(message) - - def save_frontend_messages(self, message): - self._frontend_messages.save_message(message) - def listen(self, port): self.application.listen(port) - - -class MessageStorage(ABC): - def __init__(self): - self.saved_messages = [] - - def save_message(self, message): - with suppress(KeyError, AttributeError): - if self.filter_message(message): - self.saved_messages.extend(self.transform_message(message)) - - @abstractmethod - def filter_message(self, message): - raise NotImplementedError - - def transform_message(self, message): # pylint: disable=no-self-use - yield message - - def clear(self): - self.saved_messages.clear() - - -class FrontendMessageStorage(MessageStorage): - def filter_message(self, message): - key = message['key'] - command = message.get('data', {}).get('command') - return ( - key in ('message', 'dashboard', 'queueMessages') - or key == 'ide' and command in ('select', 'read') - ) - - def transform_message(self, message): - if message['key'] == 'queueMessages': - yield from MessageSender.generate_messages_from_queue(message) - else: - yield message - - def replay_messages(self, connector): - for message in self.saved_messages: - connector.send(message) diff --git a/lib/tfw/networking/server/zmq_websocket_proxy.py b/lib/tfw/networking/server/zmq_websocket_proxy.py index 15826bd..b813dca 100644 --- a/lib/tfw/networking/server/zmq_websocket_proxy.py +++ b/lib/tfw/networking/server/zmq_websocket_proxy.py @@ -5,7 +5,7 @@ import json from tornado.websocket import WebSocketHandler -from tfw.mixins.callback_mixin import CallbackMixin +from tfw.networking.event_handlers.server_connector import Scope from tfw.config.logs import logging LOG = logging.getLogger(__name__) @@ -14,38 +14,18 @@ LOG = logging.getLogger(__name__) class ZMQWebSocketProxy(WebSocketHandler): # pylint: disable=abstract-method instances = set() - sequence_number = 0 def initialize(self, **kwargs): # pylint: disable=arguments-differ - self._event_handler_connector = kwargs['event_handler_connector'] - self._proxy_filters_and_callbacks = kwargs.get('proxy_filters_and_callbacks', {}) + self.event_handler_connector = kwargs['event_handler_connector'] + self.tfw_router = TFWRouter(self.send_to_zmq, self.send_to_websockets) - self.proxy_eventhandler_to_websocket = TFWProxy( - self.send_eventhandler_message, - self.send_websocket_message - ) - self.proxy_websocket_to_eventhandler = TFWProxy( - self.send_websocket_message, - self.send_eventhandler_message - ) + def send_to_zmq(self, message): + self.event_handler_connector.send_message(message) - self.subscribe_proxy_callbacks() - - def subscribe_proxy_callbacks(self): - eventhandler_message_handlers = self._proxy_filters_and_callbacks.get('eventhandler_message_handlers', []) - frontend_message_handlers = self._proxy_filters_and_callbacks.get('frontend_message_handlers', []) - message_handlers = self._proxy_filters_and_callbacks.get('message_handlers', []) - proxy_filters = self._proxy_filters_and_callbacks.get('proxy_filters', []) - - self.proxy_websocket_to_eventhandler.subscribe_proxy_callbacks_and_filters( - eventhandler_message_handlers + message_handlers, - proxy_filters - ) - - self.proxy_eventhandler_to_websocket.subscribe_proxy_callbacks_and_filters( - frontend_message_handlers + message_handlers, - proxy_filters - ) + @staticmethod + def send_to_websockets(message): + for instance in ZMQWebSocketProxy.instances: + instance.write_message(message) def prepare(self): ZMQWebSocketProxy.instances.add(self) @@ -54,102 +34,39 @@ class ZMQWebSocketProxy(WebSocketHandler): ZMQWebSocketProxy.instances.remove(self) def open(self, *args, **kwargs): - LOG.debug('WebSocket connection initiated') - self._event_handler_connector.register_callback(self.eventhander_callback) + LOG.debug('WebSocket connection initiated!') + self.event_handler_connector.register_callback(self.zmq_callback) - def eventhander_callback(self, message): - """ - Invoked on ZMQ messages from event handlers. - """ - self.sequence_message(message) - LOG.debug('Received on pull socket: %s', message) - self.proxy_eventhandler_to_websocket(message) - - @classmethod - def sequence_message(cls, message): - cls.sequence_number += 1 - message['seq'] = cls.sequence_number + def zmq_callback(self, message): + LOG.debug('Received on ZMQ pull socket: %s', message) + self.tfw_router.route(message) def on_message(self, message): - """ - Invoked on WS messages from frontend. - """ message = json.loads(message) - self.sequence_message(message) LOG.debug('Received on WebSocket: %s', message) - self.proxy_websocket_to_eventhandler(message) - - def send_eventhandler_message(self, message): - self._event_handler_connector.send_message(message) - - @staticmethod - def send_websocket_message(message): - for instance in ZMQWebSocketProxy.instances: - instance.write_message(message) + self.tfw_router.route(message) # much secure, very cors, wow def check_origin(self, origin): return True -class TFWProxy: - # pylint: disable=protected-access - def __init__(self, to_source, to_destination): - self.to_source = to_source - self.to_destination = to_destination +class TFWRouter: + def __init__(self, send_to_zmq, send_to_websockets): + self.send_to_zmq = send_to_zmq + self.send_to_websockets = send_to_websockets - self.proxy_filters = CallbackMixin() - self.proxy_callbacks = CallbackMixin() + def route(self, message): + scope = Scope(message.get('scope', 'zmq')) - self.proxy_filters.subscribe_callback(self.validate_message) - - self.keyhandlers = { - 'mirror': self.mirror, - 'broadcast': self.broadcast + routing_table = { + Scope.ZMQ: self.send_to_zmq, + Scope.WEBSOCKET: self.send_to_websockets, + Scope.BROADCAST: self.broadcast } - - @staticmethod - def validate_message(message): - if 'key' not in message: - raise ValueError('Invalid TFW message format!') - - def __call__(self, message): - if not self.filter_and_execute_callbacks(message): - return - - if message['key'] not in self.keyhandlers: - self.to_destination(message) - else: - handler = self.keyhandlers[message['key']] - try: - handler(message) - except KeyError: - LOG.error('Invalid "%s" message format! Ignoring.', handler.__name__) - - def filter_and_execute_callbacks(self, message): - try: - self.proxy_filters._execute_callbacks(message) - self.proxy_callbacks._execute_callbacks(message) - return True - except ValueError: - LOG.exception('Invalid TFW message received!') - return False - - def mirror(self, message): - message = message['data'] - if not self.filter_and_execute_callbacks(message): - return - LOG.debug('Mirroring message: %s', message) - self.to_source(message) + action = routing_table[scope] + action(message) def broadcast(self, message): - message = message['data'] - if not self.filter_and_execute_callbacks(message): - return - LOG.debug('Broadcasting message: %s', message) - self.to_source(message) - self.to_destination(message) - - def subscribe_proxy_callbacks_and_filters(self, proxy_callbacks, proxy_filters): - self.proxy_callbacks.subscribe_callbacks(*proxy_callbacks) - self.proxy_filters.subscribe_callbacks(*proxy_filters) + self.send_to_zmq(message) + self.send_to_websockets(message) From 2134d743c38e7ef4414b96b843be701fe71d8727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 27 May 2019 14:09:13 +0200 Subject: [PATCH 012/174] Improve TFW lib layout --- lib/tfw/components/directory_monitor.py | 2 +- .../directory_monitoring_event_handler.py | 2 +- .../directory_snapshotting_event_handler.py | 2 +- lib/tfw/components/frontend_event_handler.py | 2 +- lib/tfw/components/fsm_managing_event_handler.py | 4 ++-- lib/tfw/components/ide_event_handler.py | 2 +- lib/tfw/components/log_monitor.py | 2 +- lib/tfw/components/log_monitoring_event_handler.py | 2 +- lib/tfw/components/pipe_io_event_handler.py | 2 +- .../components/process_managing_event_handler.py | 2 +- lib/tfw/components/terminal_event_handler.py | 3 +-- lib/tfw/crypto.py | 2 +- .../__init__.py | 3 ++- .../boradcasting_event_handler.py | 5 +++-- .../event_handler_base.py | 7 +------ .../event_handlers/frontend_event_handler_base.py | 8 ++++++++ lib/tfw/{networking => event_handlers}/fsm_aware.py | 1 - .../fsm_aware_event_handler.py | 4 ++-- lib/tfw/networking/__init__.py | 6 ++++-- .../{server => }/event_handler_connector.py | 5 +++-- lib/tfw/networking/event_handlers/__init__.py | 2 -- lib/tfw/networking/message_sender.py | 2 +- lib/tfw/networking/scope.py | 7 +++++++ lib/tfw/networking/server/__init__.py | 2 -- .../{event_handlers => }/server_connector.py | 13 ++++--------- lib/tfw/server/__init__.py | 1 + lib/tfw/{networking => }/server/tfw_server.py | 5 +++-- .../{networking => }/server/zmq_websocket_proxy.py | 2 +- supervisor/tfw_server.py | 2 +- 29 files changed, 54 insertions(+), 48 deletions(-) rename lib/tfw/{event_handler_base => event_handlers}/__init__.py (67%) rename lib/tfw/{event_handler_base => event_handlers}/boradcasting_event_handler.py (87%) rename lib/tfw/{event_handler_base => event_handlers}/event_handler_base.py (93%) create mode 100644 lib/tfw/event_handlers/frontend_event_handler_base.py rename lib/tfw/{networking => event_handlers}/fsm_aware.py (99%) rename lib/tfw/{event_handler_base => event_handlers}/fsm_aware_event_handler.py (84%) rename lib/tfw/networking/{server => }/event_handler_connector.py (92%) delete mode 100644 lib/tfw/networking/event_handlers/__init__.py create mode 100644 lib/tfw/networking/scope.py delete mode 100644 lib/tfw/networking/server/__init__.py rename lib/tfw/networking/{event_handlers => }/server_connector.py (87%) create mode 100644 lib/tfw/server/__init__.py rename lib/tfw/{networking => }/server/tfw_server.py (83%) rename lib/tfw/{networking => }/server/zmq_websocket_proxy.py (97%) diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index 6aa2ca0..f4fb699 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -5,7 +5,7 @@ from functools import wraps from watchdog.events import FileSystemEventHandler as FileSystemWatchdogEventHandler -from tfw.networking.event_handlers.server_connector import ServerUplinkConnector, Scope +from tfw.networking import ServerUplinkConnector, Scope from tfw.decorators.rate_limiter import RateLimiter from tfw.mixins.observer_mixin import ObserverMixin diff --git a/lib/tfw/components/directory_monitoring_event_handler.py b/lib/tfw/components/directory_monitoring_event_handler.py index 03c4a3a..18bb8ca 100644 --- a/lib/tfw/components/directory_monitoring_event_handler.py +++ b/lib/tfw/components/directory_monitoring_event_handler.py @@ -3,7 +3,7 @@ from os.path import isdir, exists -from tfw.event_handler_base import FrontendEventHandlerBase +from tfw.event_handlers import FrontendEventHandlerBase from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin from tfw.components.directory_monitor import DirectoryMonitor from tfw.config.logs import logging diff --git a/lib/tfw/components/directory_snapshotting_event_handler.py b/lib/tfw/components/directory_snapshotting_event_handler.py index 6a285a8..d60732f 100644 --- a/lib/tfw/components/directory_snapshotting_event_handler.py +++ b/lib/tfw/components/directory_snapshotting_event_handler.py @@ -8,7 +8,7 @@ from datetime import datetime from dateutil import parser as dateparser -from tfw.event_handler_base import FrontendEventHandlerBase +from tfw.event_handlers import FrontendEventHandlerBase from tfw.components.snapshot_provider import SnapshotProvider from tfw.config import TFWENV from tfw.config.logs import logging diff --git a/lib/tfw/components/frontend_event_handler.py b/lib/tfw/components/frontend_event_handler.py index 898bb7e..73028ba 100644 --- a/lib/tfw/components/frontend_event_handler.py +++ b/lib/tfw/components/frontend_event_handler.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from contextlib import suppress from tfw.networking.message_sender import MessageSender -from tfw.event_handler_base import FrontendEventHandlerBase +from tfw.event_handlers import FrontendEventHandlerBase class FrontendEventHandler(FrontendEventHandlerBase): diff --git a/lib/tfw/components/fsm_managing_event_handler.py b/lib/tfw/components/fsm_managing_event_handler.py index 3069fc1..ac6e6fc 100644 --- a/lib/tfw/components/fsm_managing_event_handler.py +++ b/lib/tfw/components/fsm_managing_event_handler.py @@ -1,10 +1,10 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw.event_handler_base import FrontendEventHandlerBase +from tfw.event_handlers import FrontendEventHandlerBase from tfw.crypto import KeyManager, sign_message, verify_message from tfw.config.logs import logging -from tfw.networking.event_handlers.server_connector import Scope +from tfw.networking import Scope LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/ide_event_handler.py b/lib/tfw/components/ide_event_handler.py index 534de6a..2632b45 100644 --- a/lib/tfw/components/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler.py @@ -6,7 +6,7 @@ from glob import glob from fnmatch import fnmatchcase from typing import Iterable -from tfw.event_handler_base import FrontendEventHandlerBase +from tfw.event_handlers import FrontendEventHandlerBase from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin from tfw.components.directory_monitor import DirectoryMonitor from tfw.config.logs import logging diff --git a/lib/tfw/components/log_monitor.py b/lib/tfw/components/log_monitor.py index 2ee6d96..b8a0b49 100644 --- a/lib/tfw/components/log_monitor.py +++ b/lib/tfw/components/log_monitor.py @@ -6,7 +6,7 @@ from os.path import dirname from watchdog.events import PatternMatchingEventHandler as PatternMatchingWatchdogEventHandler -from tfw.networking.event_handlers.server_connector import ServerUplinkConnector, Scope +from tfw.networking import ServerUplinkConnector, Scope from tfw.decorators.rate_limiter import RateLimiter from tfw.mixins.observer_mixin import ObserverMixin from tfw.mixins.supervisor_mixin import SupervisorLogMixin diff --git a/lib/tfw/components/log_monitoring_event_handler.py b/lib/tfw/components/log_monitoring_event_handler.py index d384323..23fe5b9 100644 --- a/lib/tfw/components/log_monitoring_event_handler.py +++ b/lib/tfw/components/log_monitoring_event_handler.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw.event_handler_base import FrontendEventHandlerBase +from tfw.event_handlers import FrontendEventHandlerBase from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin from tfw.components.log_monitor import LogMonitor from tfw.config.logs import logging diff --git a/lib/tfw/components/pipe_io_event_handler.py b/lib/tfw/components/pipe_io_event_handler.py index 37b5b89..e35455e 100644 --- a/lib/tfw/components/pipe_io_event_handler.py +++ b/lib/tfw/components/pipe_io_event_handler.py @@ -9,7 +9,7 @@ from secrets import token_urlsafe from threading import Thread from contextlib import suppress -from tfw.event_handler_base import EventHandlerBase +from tfw.event_handlers import EventHandlerBase from tfw.config.logs import logging from .pipe_io_server import PipeIOServer, terminate_process_on_failure diff --git a/lib/tfw/components/process_managing_event_handler.py b/lib/tfw/components/process_managing_event_handler.py index a87bdd6..f6489e6 100644 --- a/lib/tfw/components/process_managing_event_handler.py +++ b/lib/tfw/components/process_managing_event_handler.py @@ -3,7 +3,7 @@ from xmlrpc.client import Fault as SupervisorFault -from tfw.event_handler_base import FrontendEventHandlerBase +from tfw.event_handlers import FrontendEventHandlerBase from tfw.mixins.supervisor_mixin import SupervisorMixin, SupervisorLogMixin from tfw.components.directory_monitor import with_monitor_paused from tfw.config.logs import logging diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/components/terminal_event_handler.py index f45bc8a..3427c4f 100644 --- a/lib/tfw/components/terminal_event_handler.py +++ b/lib/tfw/components/terminal_event_handler.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw.event_handler_base import FrontendEventHandlerBase +from tfw.event_handlers import FrontendEventHandlerBase from tfw.components.terminado_mini_server import TerminadoMiniServer from tfw.config import TFWENV from tfw.config.logs import logging @@ -50,7 +50,6 @@ class TerminalEventHandler(FrontendEventHandlerBase): return self._historymonitor def handle_event(self, message): - LOG.debug('TerminadoEventHandler received event: %s', message) try: data = message['data'] message['data'] = self.commands[data['command']](data) diff --git a/lib/tfw/crypto.py b/lib/tfw/crypto.py index 0b37893..344abee 100644 --- a/lib/tfw/crypto.py +++ b/lib/tfw/crypto.py @@ -14,7 +14,7 @@ from cryptography.hazmat.primitives.hashes import SHA256 from cryptography.hazmat.primitives.hmac import HMAC as _HMAC from cryptography.exceptions import InvalidSignature -from tfw.networking.serialization import message_bytes +from tfw.networking import message_bytes from tfw.decorators.lazy_property import lazy_property from tfw.config import TFWENV diff --git a/lib/tfw/event_handler_base/__init__.py b/lib/tfw/event_handlers/__init__.py similarity index 67% rename from lib/tfw/event_handler_base/__init__.py rename to lib/tfw/event_handlers/__init__.py index 65ca6e1..858b781 100644 --- a/lib/tfw/event_handler_base/__init__.py +++ b/lib/tfw/event_handlers/__init__.py @@ -1,6 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from .event_handler_base import EventHandlerBase, FrontendEventHandlerBase +from .event_handler_base import EventHandlerBase +from .frontend_event_handler_base import FrontendEventHandlerBase from .boradcasting_event_handler import BroadcastingEventHandler from .fsm_aware_event_handler import FSMAwareEventHandler diff --git a/lib/tfw/event_handler_base/boradcasting_event_handler.py b/lib/tfw/event_handlers/boradcasting_event_handler.py similarity index 87% rename from lib/tfw/event_handler_base/boradcasting_event_handler.py rename to lib/tfw/event_handlers/boradcasting_event_handler.py index cfda694..c6f61a9 100644 --- a/lib/tfw/event_handler_base/boradcasting_event_handler.py +++ b/lib/tfw/event_handlers/boradcasting_event_handler.py @@ -3,10 +3,11 @@ from abc import ABC -from tfw.event_handler_base.event_handler_base import EventHandlerBase -from tfw.networking.event_handlers.server_connector import Scope +from tfw.networking import Scope from tfw.crypto import message_checksum +from .event_handler_base import EventHandlerBase + class BroadcastingEventHandler(EventHandlerBase, ABC): # pylint: disable=abstract-method diff --git a/lib/tfw/event_handler_base/event_handler_base.py b/lib/tfw/event_handlers/event_handler_base.py similarity index 93% rename from lib/tfw/event_handler_base/event_handler_base.py rename to lib/tfw/event_handlers/event_handler_base.py index f51c668..bb3155f 100644 --- a/lib/tfw/event_handler_base/event_handler_base.py +++ b/lib/tfw/event_handlers/event_handler_base.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from inspect import currentframe from typing import Iterable -from tfw.networking.event_handlers.server_connector import ServerConnector, Scope +from tfw.networking import ServerConnector from tfw.config.logs import logging LOG = logging.getLogger(__name__) @@ -131,8 +131,3 @@ class EventHandlerBase(ABC): instance for instance in locals_values if isinstance(instance, cls) } - - -class FrontendEventHandlerBase(EventHandlerBase): # pylint: disable=abstract-method - def send_message(self, message): - self.server_connector.send_message(message, Scope.WEBSOCKET) diff --git a/lib/tfw/event_handlers/frontend_event_handler_base.py b/lib/tfw/event_handlers/frontend_event_handler_base.py new file mode 100644 index 0000000..6990632 --- /dev/null +++ b/lib/tfw/event_handlers/frontend_event_handler_base.py @@ -0,0 +1,8 @@ +from tfw.networking import Scope + +from .event_handler_base import EventHandlerBase + + +class FrontendEventHandlerBase(EventHandlerBase): # pylint: disable=abstract-method + def send_message(self, message): + self.server_connector.send_message(message, Scope.WEBSOCKET) diff --git a/lib/tfw/networking/fsm_aware.py b/lib/tfw/event_handlers/fsm_aware.py similarity index 99% rename from lib/tfw/networking/fsm_aware.py rename to lib/tfw/event_handlers/fsm_aware.py index cb2b287..bec3633 100644 --- a/lib/tfw/networking/fsm_aware.py +++ b/lib/tfw/event_handlers/fsm_aware.py @@ -2,7 +2,6 @@ # All Rights Reserved. See LICENSE file for details. from tfw.crypto import KeyManager, verify_message - from tfw.config.logs import logging LOG = logging.getLogger(__name__) diff --git a/lib/tfw/event_handler_base/fsm_aware_event_handler.py b/lib/tfw/event_handlers/fsm_aware_event_handler.py similarity index 84% rename from lib/tfw/event_handler_base/fsm_aware_event_handler.py rename to lib/tfw/event_handlers/fsm_aware_event_handler.py index 01d6360..60313df 100644 --- a/lib/tfw/event_handler_base/fsm_aware_event_handler.py +++ b/lib/tfw/event_handlers/fsm_aware_event_handler.py @@ -3,8 +3,8 @@ from abc import ABC -from tfw.event_handler_base.event_handler_base import EventHandlerBase -from tfw.networking.fsm_aware import FSMAware +from .event_handler_base import EventHandlerBase +from .fsm_aware import FSMAware class FSMAwareEventHandler(EventHandlerBase, FSMAware, ABC): diff --git a/lib/tfw/networking/__init__.py b/lib/tfw/networking/__init__.py index 500dd7a..7772eb6 100644 --- a/lib/tfw/networking/__init__.py +++ b/lib/tfw/networking/__init__.py @@ -1,6 +1,8 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +from .serialization import serialize_tfw_msg, deserialize_tfw_msg, with_deserialize_tfw_msg, message_bytes +from .server_connector import ServerUplinkConnector, ServerDownlinkConnector, ServerConnector +from .event_handler_connector import EventHandlerConnector from .message_sender import MessageSender -from .event_handlers.server_connector import ServerUplinkConnector as TFWServerConnector -from .server.tfw_server import TFWServer +from .scope import Scope diff --git a/lib/tfw/networking/server/event_handler_connector.py b/lib/tfw/networking/event_handler_connector.py similarity index 92% rename from lib/tfw/networking/server/event_handler_connector.py rename to lib/tfw/networking/event_handler_connector.py index 7b56a88..67ca7ba 100644 --- a/lib/tfw/networking/server/event_handler_connector.py +++ b/lib/tfw/networking/event_handler_connector.py @@ -4,11 +4,12 @@ import zmq from zmq.eventloop.zmqstream import ZMQStream -from tfw.networking.zmq_connector_base import ZMQConnectorBase -from tfw.networking.serialization import serialize_tfw_msg, with_deserialize_tfw_msg from tfw.config import TFWENV from tfw.config.logs import logging +from .serialization import serialize_tfw_msg, with_deserialize_tfw_msg +from .zmq_connector_base import ZMQConnectorBase + LOG = logging.getLogger(__name__) diff --git a/lib/tfw/networking/event_handlers/__init__.py b/lib/tfw/networking/event_handlers/__init__.py deleted file mode 100644 index db64b25..0000000 --- a/lib/tfw/networking/event_handlers/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. diff --git a/lib/tfw/networking/message_sender.py b/lib/tfw/networking/message_sender.py index 1232aaa..10dd462 100644 --- a/lib/tfw/networking/message_sender.py +++ b/lib/tfw/networking/message_sender.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw.networking.event_handlers.server_connector import ServerUplinkConnector +from .server_connector import ServerUplinkConnector class MessageSender: diff --git a/lib/tfw/networking/scope.py b/lib/tfw/networking/scope.py new file mode 100644 index 0000000..639461b --- /dev/null +++ b/lib/tfw/networking/scope.py @@ -0,0 +1,7 @@ +from enum import Enum + + +class Scope(Enum): + ZMQ = 'zmq' + WEBSOCKET = 'websocket' + BROADCAST = 'broadcast' diff --git a/lib/tfw/networking/server/__init__.py b/lib/tfw/networking/server/__init__.py deleted file mode 100644 index db64b25..0000000 --- a/lib/tfw/networking/server/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. diff --git a/lib/tfw/networking/event_handlers/server_connector.py b/lib/tfw/networking/server_connector.py similarity index 87% rename from lib/tfw/networking/event_handlers/server_connector.py rename to lib/tfw/networking/server_connector.py index 73b04e0..c0e2bf1 100644 --- a/lib/tfw/networking/event_handlers/server_connector.py +++ b/lib/tfw/networking/server_connector.py @@ -2,16 +2,17 @@ # All Rights Reserved. See LICENSE file for details. from functools import partial -from enum import Enum import zmq from zmq.eventloop.zmqstream import ZMQStream -from tfw.networking.zmq_connector_base import ZMQConnectorBase -from tfw.networking.serialization import serialize_tfw_msg, with_deserialize_tfw_msg from tfw.config import TFWENV from tfw.config.logs import logging +from .scope import Scope +from .serialization import serialize_tfw_msg, with_deserialize_tfw_msg +from .zmq_connector_base import ZMQConnectorBase + LOG = logging.getLogger(__name__) @@ -34,12 +35,6 @@ class ServerDownlinkConnector(ZMQConnectorBase): self._zmq_sub_stream.close() -class Scope(Enum): - ZMQ = 'zmq' - WEBSOCKET = 'websocket' - BROADCAST = 'broadcast' - - class ServerUplinkConnector(ZMQConnectorBase): def __init__(self, zmq_context=None): super(ServerUplinkConnector, self).__init__(zmq_context) diff --git a/lib/tfw/server/__init__.py b/lib/tfw/server/__init__.py new file mode 100644 index 0000000..e2c01a9 --- /dev/null +++ b/lib/tfw/server/__init__.py @@ -0,0 +1 @@ +from .tfw_server import TFWServer diff --git a/lib/tfw/networking/server/tfw_server.py b/lib/tfw/server/tfw_server.py similarity index 83% rename from lib/tfw/networking/server/tfw_server.py rename to lib/tfw/server/tfw_server.py index ed28e62..a4cdd3f 100644 --- a/lib/tfw/networking/server/tfw_server.py +++ b/lib/tfw/server/tfw_server.py @@ -3,10 +3,11 @@ from tornado.web import Application -from tfw.networking.server.zmq_websocket_proxy import ZMQWebSocketProxy -from tfw.networking.server.event_handler_connector import EventHandlerConnector +from tfw.networking import EventHandlerConnector from tfw.config.logs import logging +from .zmq_websocket_proxy import ZMQWebSocketProxy + LOG = logging.getLogger(__name__) diff --git a/lib/tfw/networking/server/zmq_websocket_proxy.py b/lib/tfw/server/zmq_websocket_proxy.py similarity index 97% rename from lib/tfw/networking/server/zmq_websocket_proxy.py rename to lib/tfw/server/zmq_websocket_proxy.py index b813dca..b820f85 100644 --- a/lib/tfw/networking/server/zmq_websocket_proxy.py +++ b/lib/tfw/server/zmq_websocket_proxy.py @@ -5,7 +5,7 @@ import json from tornado.websocket import WebSocketHandler -from tfw.networking.event_handlers.server_connector import Scope +from tfw.networking import Scope from tfw.config.logs import logging LOG = logging.getLogger(__name__) diff --git a/supervisor/tfw_server.py b/supervisor/tfw_server.py index 78766ac..4cb8bd6 100644 --- a/supervisor/tfw_server.py +++ b/supervisor/tfw_server.py @@ -1,6 +1,6 @@ from tornado.ioloop import IOLoop -from tfw.networking import TFWServer +from tfw.server import TFWServer from tfw.config import TFWENV From 82df8a80654d3b9c938a4903bb0a4ba0077d9b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 27 May 2019 14:16:51 +0200 Subject: [PATCH 013/174] Handle pylint suggestions --- .pylintrc | 1 + lib/tfw/event_handlers/event_handler_base.py | 1 - lib/tfw/event_handlers/fsm_aware.py | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.pylintrc b/.pylintrc index fd195bc..1cbae5d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,6 +6,7 @@ disable = missing-docstring, too-few-public-methods, invalid-name [SIMILARITIES] +min-similarity-lines=6 ignore-comments=yes ignore-docstrings=yes ignore-imports=yes diff --git a/lib/tfw/event_handlers/event_handler_base.py b/lib/tfw/event_handlers/event_handler_base.py index bb3155f..95300fc 100644 --- a/lib/tfw/event_handlers/event_handler_base.py +++ b/lib/tfw/event_handlers/event_handler_base.py @@ -118,7 +118,6 @@ class EventHandlerBase(ABC): Perform cleanup actions such as releasing database connections and stuff like that. """ - pass @classmethod def get_local_instances(cls): diff --git a/lib/tfw/event_handlers/fsm_aware.py b/lib/tfw/event_handlers/fsm_aware.py index bec3633..13cea43 100644 --- a/lib/tfw/event_handlers/fsm_aware.py +++ b/lib/tfw/event_handlers/fsm_aware.py @@ -42,4 +42,3 @@ class FSMAware: :param kwargs: fsm_update 'data' field """ - pass From e44a99fa6dae9944b410875e75cb98ad83d448d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 27 May 2019 20:02:09 +0200 Subject: [PATCH 014/174] Remove ZMQConnectorBase --- lib/tfw/networking/event_handler_connector.py | 19 ++++++++++--------- lib/tfw/networking/server_connector.py | 19 ++++++++++--------- lib/tfw/networking/zmq_connector_base.py | 9 --------- 3 files changed, 20 insertions(+), 27 deletions(-) delete mode 100644 lib/tfw/networking/zmq_connector_base.py diff --git a/lib/tfw/networking/event_handler_connector.py b/lib/tfw/networking/event_handler_connector.py index 67ca7ba..03f2513 100644 --- a/lib/tfw/networking/event_handler_connector.py +++ b/lib/tfw/networking/event_handler_connector.py @@ -8,15 +8,13 @@ from tfw.config import TFWENV from tfw.config.logs import logging from .serialization import serialize_tfw_msg, with_deserialize_tfw_msg -from .zmq_connector_base import ZMQConnectorBase LOG = logging.getLogger(__name__) -class EventHandlerDownlinkConnector(ZMQConnectorBase): - def __init__(self, zmq_context=None): - super(EventHandlerDownlinkConnector, self).__init__(zmq_context) - self._zmq_pull_socket = self._zmq_context.socket(zmq.PULL) +class EventHandlerDownlinkConnector(): + def __init__(self): + self._zmq_pull_socket = zmq.Context.instance().socket(zmq.PULL) self._zmq_pull_socket.setsockopt(zmq.RCVHWM, 0) self._zmq_pull_stream = ZMQStream(self._zmq_pull_socket) address = f'tcp://*:{TFWENV.RECEIVER_PORT}' @@ -31,10 +29,9 @@ class EventHandlerDownlinkConnector(ZMQConnectorBase): self._zmq_pull_stream.close() -class EventHandlerUplinkConnector(ZMQConnectorBase): - def __init__(self, zmq_context=None): - super(EventHandlerUplinkConnector, self).__init__(zmq_context) - self._zmq_pub_socket = self._zmq_context.socket(zmq.PUB) +class EventHandlerUplinkConnector(): + def __init__(self): + self._zmq_pub_socket = zmq.Context.instance().socket(zmq.PUB) self._zmq_pub_socket.setsockopt(zmq.SNDHWM, 0) address = f'tcp://*:{TFWENV.PUBLISHER_PORT}' self._zmq_pub_socket.bind(address) @@ -48,6 +45,10 @@ class EventHandlerUplinkConnector(ZMQConnectorBase): class EventHandlerConnector(EventHandlerDownlinkConnector, EventHandlerUplinkConnector): + def __init__(self): + EventHandlerDownlinkConnector.__init__(self) + EventHandlerUplinkConnector.__init__(self) + def close(self): EventHandlerDownlinkConnector.close(self) EventHandlerUplinkConnector.close(self) diff --git a/lib/tfw/networking/server_connector.py b/lib/tfw/networking/server_connector.py index c0e2bf1..da84289 100644 --- a/lib/tfw/networking/server_connector.py +++ b/lib/tfw/networking/server_connector.py @@ -11,15 +11,13 @@ from tfw.config.logs import logging from .scope import Scope from .serialization import serialize_tfw_msg, with_deserialize_tfw_msg -from .zmq_connector_base import ZMQConnectorBase LOG = logging.getLogger(__name__) -class ServerDownlinkConnector(ZMQConnectorBase): - def __init__(self, zmq_context=None): - super(ServerDownlinkConnector, self).__init__(zmq_context) - self._zmq_sub_socket = self._zmq_context.socket(zmq.SUB) +class ServerDownlinkConnector(): + def __init__(self): + self._zmq_sub_socket = zmq.Context.instance().socket(zmq.SUB) self._zmq_sub_socket.connect(f'tcp://localhost:{TFWENV.PUBLISHER_PORT}') self._zmq_sub_socket.setsockopt(zmq.RCVHWM, 0) self._zmq_sub_stream = ZMQStream(self._zmq_sub_socket) @@ -35,10 +33,9 @@ class ServerDownlinkConnector(ZMQConnectorBase): self._zmq_sub_stream.close() -class ServerUplinkConnector(ZMQConnectorBase): - def __init__(self, zmq_context=None): - super(ServerUplinkConnector, self).__init__(zmq_context) - self._zmq_push_socket = self._zmq_context.socket(zmq.PUSH) +class ServerUplinkConnector(): + def __init__(self): + self._zmq_push_socket = zmq.Context.instance().socket(zmq.PUSH) self._zmq_push_socket.connect(f'tcp://localhost:{TFWENV.RECEIVER_PORT}') self._zmq_push_socket.setsockopt(zmq.SNDHWM, 0) @@ -51,6 +48,10 @@ class ServerUplinkConnector(ZMQConnectorBase): class ServerConnector(ServerUplinkConnector, ServerDownlinkConnector): + def __init__(self): + ServerUplinkConnector.__init__(self) + ServerDownlinkConnector.__init__(self) + def close(self): ServerUplinkConnector.close(self) ServerDownlinkConnector.close(self) diff --git a/lib/tfw/networking/zmq_connector_base.py b/lib/tfw/networking/zmq_connector_base.py deleted file mode 100644 index 47a850e..0000000 --- a/lib/tfw/networking/zmq_connector_base.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -import zmq - - -class ZMQConnectorBase: - def __init__(self, zmq_context=None): - self._zmq_context = zmq_context or zmq.Context.instance() From f151ecfbac378187856c0d3c0af6fb835dd30933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 27 May 2019 20:11:03 +0200 Subject: [PATCH 015/174] Improve ZMQ port envvar names --- Dockerfile | 4 ++-- lib/tfw/networking/event_handler_connector.py | 4 ++-- lib/tfw/networking/server_connector.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 22237e8..94eee57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,8 +23,8 @@ ENV TFW_PUBLIC_PORT=8888 \ TFW_LOGIN_APP_PORT=6666 \ TFW_TERMINADO_PORT=7878 \ TFW_SUPERVISOR_HTTP_PORT=9001 \ - TFW_PUBLISHER_PORT=7654 \ - TFW_RECEIVER_PORT=8765 + TFW_PUB_PORT=7654 \ + TFW_PULL_PORT=8765 EXPOSE ${TFW_PUBLIC_PORT} diff --git a/lib/tfw/networking/event_handler_connector.py b/lib/tfw/networking/event_handler_connector.py index 03f2513..bf2c9ae 100644 --- a/lib/tfw/networking/event_handler_connector.py +++ b/lib/tfw/networking/event_handler_connector.py @@ -17,7 +17,7 @@ class EventHandlerDownlinkConnector(): self._zmq_pull_socket = zmq.Context.instance().socket(zmq.PULL) self._zmq_pull_socket.setsockopt(zmq.RCVHWM, 0) self._zmq_pull_stream = ZMQStream(self._zmq_pull_socket) - address = f'tcp://*:{TFWENV.RECEIVER_PORT}' + address = f'tcp://*:{TFWENV.PULL_PORT}' self._zmq_pull_socket.bind(address) LOG.debug('Pull socket bound to %s', address) @@ -33,7 +33,7 @@ class EventHandlerUplinkConnector(): def __init__(self): self._zmq_pub_socket = zmq.Context.instance().socket(zmq.PUB) self._zmq_pub_socket.setsockopt(zmq.SNDHWM, 0) - address = f'tcp://*:{TFWENV.PUBLISHER_PORT}' + address = f'tcp://*:{TFWENV.PUB_PORT}' self._zmq_pub_socket.bind(address) LOG.debug('Pub socket bound to %s', address) diff --git a/lib/tfw/networking/server_connector.py b/lib/tfw/networking/server_connector.py index da84289..600cb9a 100644 --- a/lib/tfw/networking/server_connector.py +++ b/lib/tfw/networking/server_connector.py @@ -18,7 +18,7 @@ LOG = logging.getLogger(__name__) class ServerDownlinkConnector(): def __init__(self): self._zmq_sub_socket = zmq.Context.instance().socket(zmq.SUB) - self._zmq_sub_socket.connect(f'tcp://localhost:{TFWENV.PUBLISHER_PORT}') + self._zmq_sub_socket.connect(f'tcp://localhost:{TFWENV.PUB_PORT}') self._zmq_sub_socket.setsockopt(zmq.RCVHWM, 0) self._zmq_sub_stream = ZMQStream(self._zmq_sub_socket) @@ -36,7 +36,7 @@ class ServerDownlinkConnector(): class ServerUplinkConnector(): def __init__(self): self._zmq_push_socket = zmq.Context.instance().socket(zmq.PUSH) - self._zmq_push_socket.connect(f'tcp://localhost:{TFWENV.RECEIVER_PORT}') + self._zmq_push_socket.connect(f'tcp://localhost:{TFWENV.PULL_PORT}') self._zmq_push_socket.setsockopt(zmq.SNDHWM, 0) def send_message(self, message, scope=Scope.ZMQ): From c8e98af5161d4a6277d022b04b4fdcd9d864d866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 4 Jun 2019 13:58:03 +0200 Subject: [PATCH 016/174] Improve module dependencies by moving port envvars out of tfw.networking --- lib/tfw/components/__init__.py | 1 + lib/tfw/components/directory_monitor.py | 5 +++-- lib/tfw/components/frontend_event_handler.py | 3 ++- lib/tfw/components/log_monitor.py | 5 +++-- .../message_sender.py | 4 ++-- lib/tfw/event_handlers/__init__.py | 1 + lib/tfw/event_handlers/event_handler_base.py | 5 +++-- .../event_handlers/tfw_server_connector.py | 19 +++++++++++++++++ lib/tfw/networking/__init__.py | 1 - lib/tfw/networking/event_handler_connector.py | 21 ++++++++----------- lib/tfw/networking/server_connector.py | 19 ++++++++--------- lib/tfw/server/tfw_server.py | 14 ++++++++----- ...ocket_proxy.py => zmq_websocket_router.py} | 12 +++++------ supervisor/tfw_server.py | 3 +-- 14 files changed, 68 insertions(+), 45 deletions(-) rename lib/tfw/{networking => components}/message_sender.py (93%) create mode 100644 lib/tfw/event_handlers/tfw_server_connector.py rename lib/tfw/server/{zmq_websocket_proxy.py => zmq_websocket_router.py} (88%) diff --git a/lib/tfw/components/__init__.py b/lib/tfw/components/__init__.py index 7d2ebc0..b955ad8 100644 --- a/lib/tfw/components/__init__.py +++ b/lib/tfw/components/__init__.py @@ -15,3 +15,4 @@ from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHa from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler from .commands_equal import CommandsEqual from .frontend_event_handler import FrontendEventHandler +from .message_sender import MessageSender diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index f4fb699..7144f0e 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -5,7 +5,8 @@ from functools import wraps from watchdog.events import FileSystemEventHandler as FileSystemWatchdogEventHandler -from tfw.networking import ServerUplinkConnector, Scope +from tfw.networking import Scope +from tfw.event_handlers import TFWServerUplinkConnector from tfw.decorators.rate_limiter import RateLimiter from tfw.mixins.observer_mixin import ObserverMixin @@ -47,7 +48,7 @@ class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler): def __init__(self, ide_key): super().__init__() self.ide_key = ide_key - self.uplink = ServerUplinkConnector() + self.uplink = TFWServerUplinkConnector() self._paused = False self.ignore = 0 diff --git a/lib/tfw/components/frontend_event_handler.py b/lib/tfw/components/frontend_event_handler.py index 73028ba..1a76abe 100644 --- a/lib/tfw/components/frontend_event_handler.py +++ b/lib/tfw/components/frontend_event_handler.py @@ -1,9 +1,10 @@ from abc import ABC, abstractmethod from contextlib import suppress -from tfw.networking.message_sender import MessageSender from tfw.event_handlers import FrontendEventHandlerBase +from .message_sender import MessageSender + class FrontendEventHandler(FrontendEventHandlerBase): def __init__(self): diff --git a/lib/tfw/components/log_monitor.py b/lib/tfw/components/log_monitor.py index b8a0b49..14591e8 100644 --- a/lib/tfw/components/log_monitor.py +++ b/lib/tfw/components/log_monitor.py @@ -6,7 +6,8 @@ from os.path import dirname from watchdog.events import PatternMatchingEventHandler as PatternMatchingWatchdogEventHandler -from tfw.networking import ServerUplinkConnector, Scope +from tfw.networking import Scope +from tfw.event_handlers import TFWServerUplinkConnector from tfw.decorators.rate_limiter import RateLimiter from tfw.mixins.observer_mixin import ObserverMixin from tfw.mixins.supervisor_mixin import SupervisorLogMixin @@ -38,7 +39,7 @@ class SendLogWatchdogEventHandler(PatternMatchingWatchdogEventHandler, Superviso self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile'] ]) - self.uplink = ServerUplinkConnector() + self.uplink = TFWServerUplinkConnector() self.log_tail = log_tail @property diff --git a/lib/tfw/networking/message_sender.py b/lib/tfw/components/message_sender.py similarity index 93% rename from lib/tfw/networking/message_sender.py rename to lib/tfw/components/message_sender.py index 10dd462..4f35baa 100644 --- a/lib/tfw/networking/message_sender.py +++ b/lib/tfw/components/message_sender.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from .server_connector import ServerUplinkConnector +from tfw.event_handlers import TFWServerUplinkConnector class MessageSender: @@ -9,7 +9,7 @@ class MessageSender: Provides mechanisms to send messages to our frontend messaging component. """ def __init__(self): - self.server_connector = ServerUplinkConnector() + self.server_connector = TFWServerUplinkConnector() self.key = 'message' self.queue_key = 'queueMessages' diff --git a/lib/tfw/event_handlers/__init__.py b/lib/tfw/event_handlers/__init__.py index 858b781..0327aef 100644 --- a/lib/tfw/event_handlers/__init__.py +++ b/lib/tfw/event_handlers/__init__.py @@ -5,3 +5,4 @@ from .event_handler_base import EventHandlerBase from .frontend_event_handler_base import FrontendEventHandlerBase from .boradcasting_event_handler import BroadcastingEventHandler from .fsm_aware_event_handler import FSMAwareEventHandler +from .tfw_server_connector import TFWServerUplinkConnector, TFWServerConnector diff --git a/lib/tfw/event_handlers/event_handler_base.py b/lib/tfw/event_handlers/event_handler_base.py index 95300fc..0cc0a9e 100644 --- a/lib/tfw/event_handlers/event_handler_base.py +++ b/lib/tfw/event_handlers/event_handler_base.py @@ -5,9 +5,10 @@ from abc import ABC, abstractmethod from inspect import currentframe from typing import Iterable -from tfw.networking import ServerConnector from tfw.config.logs import logging +from .tfw_server_connector import TFWServerConnector + LOG = logging.getLogger(__name__) @@ -19,7 +20,7 @@ class EventHandlerBase(ABC): Derived classes must implement the handle_event() method """ def __init__(self, key): - self.server_connector = ServerConnector() + self.server_connector = TFWServerConnector() self.keys = [] if isinstance(key, str): self.keys.append(key) diff --git a/lib/tfw/event_handlers/tfw_server_connector.py b/lib/tfw/event_handlers/tfw_server_connector.py new file mode 100644 index 0000000..161a6cb --- /dev/null +++ b/lib/tfw/event_handlers/tfw_server_connector.py @@ -0,0 +1,19 @@ +from functools import partial + +from tfw.networking import ServerUplinkConnector, ServerConnector +from tfw.config import TFWENV + + +UPLINK_CONN_ADDR = f'tcp://localhost:{TFWENV.PULL_PORT}' +DOWNLINK_CONN_ADDR = f'tcp://localhost:{TFWENV.PUB_PORT}' + + +TFWServerUplinkConnector = partial( + ServerUplinkConnector, + connect_addr=UPLINK_CONN_ADDR +) +TFWServerConnector = partial( + ServerConnector, + downlink_connect_addr=DOWNLINK_CONN_ADDR, + uplink_connect_addr=UPLINK_CONN_ADDR +) diff --git a/lib/tfw/networking/__init__.py b/lib/tfw/networking/__init__.py index 7772eb6..49e896d 100644 --- a/lib/tfw/networking/__init__.py +++ b/lib/tfw/networking/__init__.py @@ -4,5 +4,4 @@ from .serialization import serialize_tfw_msg, deserialize_tfw_msg, with_deserialize_tfw_msg, message_bytes from .server_connector import ServerUplinkConnector, ServerDownlinkConnector, ServerConnector from .event_handler_connector import EventHandlerConnector -from .message_sender import MessageSender from .scope import Scope diff --git a/lib/tfw/networking/event_handler_connector.py b/lib/tfw/networking/event_handler_connector.py index bf2c9ae..378db11 100644 --- a/lib/tfw/networking/event_handler_connector.py +++ b/lib/tfw/networking/event_handler_connector.py @@ -4,7 +4,6 @@ import zmq from zmq.eventloop.zmqstream import ZMQStream -from tfw.config import TFWENV from tfw.config.logs import logging from .serialization import serialize_tfw_msg, with_deserialize_tfw_msg @@ -13,13 +12,12 @@ LOG = logging.getLogger(__name__) class EventHandlerDownlinkConnector(): - def __init__(self): + def __init__(self, bind_addr): self._zmq_pull_socket = zmq.Context.instance().socket(zmq.PULL) self._zmq_pull_socket.setsockopt(zmq.RCVHWM, 0) self._zmq_pull_stream = ZMQStream(self._zmq_pull_socket) - address = f'tcp://*:{TFWENV.PULL_PORT}' - self._zmq_pull_socket.bind(address) - LOG.debug('Pull socket bound to %s', address) + self._zmq_pull_socket.bind(bind_addr) + LOG.debug('Pull socket bound to %s', bind_addr) def register_callback(self, callback): callback = with_deserialize_tfw_msg(callback) @@ -30,12 +28,11 @@ class EventHandlerDownlinkConnector(): class EventHandlerUplinkConnector(): - def __init__(self): + def __init__(self, bind_addr): self._zmq_pub_socket = zmq.Context.instance().socket(zmq.PUB) self._zmq_pub_socket.setsockopt(zmq.SNDHWM, 0) - address = f'tcp://*:{TFWENV.PUB_PORT}' - self._zmq_pub_socket.bind(address) - LOG.debug('Pub socket bound to %s', address) + self._zmq_pub_socket.bind(bind_addr) + LOG.debug('Pub socket bound to %s', bind_addr) def send_message(self, message: dict): self._zmq_pub_socket.send_multipart(serialize_tfw_msg(message)) @@ -45,9 +42,9 @@ class EventHandlerUplinkConnector(): class EventHandlerConnector(EventHandlerDownlinkConnector, EventHandlerUplinkConnector): - def __init__(self): - EventHandlerDownlinkConnector.__init__(self) - EventHandlerUplinkConnector.__init__(self) + def __init__(self, downlink_bind_addr, uplink_bind_addr): + EventHandlerDownlinkConnector.__init__(self, downlink_bind_addr) + EventHandlerUplinkConnector.__init__(self, uplink_bind_addr) def close(self): EventHandlerDownlinkConnector.close(self) diff --git a/lib/tfw/networking/server_connector.py b/lib/tfw/networking/server_connector.py index 600cb9a..6bee301 100644 --- a/lib/tfw/networking/server_connector.py +++ b/lib/tfw/networking/server_connector.py @@ -6,7 +6,6 @@ from functools import partial import zmq from zmq.eventloop.zmqstream import ZMQStream -from tfw.config import TFWENV from tfw.config.logs import logging from .scope import Scope @@ -16,9 +15,9 @@ LOG = logging.getLogger(__name__) class ServerDownlinkConnector(): - def __init__(self): + def __init__(self, connect_addr): self._zmq_sub_socket = zmq.Context.instance().socket(zmq.SUB) - self._zmq_sub_socket.connect(f'tcp://localhost:{TFWENV.PUB_PORT}') + self._zmq_sub_socket.connect(connect_addr) self._zmq_sub_socket.setsockopt(zmq.RCVHWM, 0) self._zmq_sub_stream = ZMQStream(self._zmq_sub_socket) @@ -34,9 +33,9 @@ class ServerDownlinkConnector(): class ServerUplinkConnector(): - def __init__(self): + def __init__(self, connect_addr): self._zmq_push_socket = zmq.Context.instance().socket(zmq.PUSH) - self._zmq_push_socket.connect(f'tcp://localhost:{TFWENV.PULL_PORT}') + self._zmq_push_socket.connect(connect_addr) self._zmq_push_socket.setsockopt(zmq.SNDHWM, 0) def send_message(self, message, scope=Scope.ZMQ): @@ -47,11 +46,11 @@ class ServerUplinkConnector(): self._zmq_push_socket.close() -class ServerConnector(ServerUplinkConnector, ServerDownlinkConnector): - def __init__(self): - ServerUplinkConnector.__init__(self) - ServerDownlinkConnector.__init__(self) +class ServerConnector(ServerDownlinkConnector, ServerUplinkConnector): + def __init__(self, downlink_connect_addr, uplink_connect_addr): + ServerDownlinkConnector.__init__(self, downlink_connect_addr) + ServerUplinkConnector.__init__(self, uplink_connect_addr) def close(self): - ServerUplinkConnector.close(self) ServerDownlinkConnector.close(self) + ServerUplinkConnector.close(self) diff --git a/lib/tfw/server/tfw_server.py b/lib/tfw/server/tfw_server.py index a4cdd3f..ad2d37e 100644 --- a/lib/tfw/server/tfw_server.py +++ b/lib/tfw/server/tfw_server.py @@ -4,9 +4,10 @@ from tornado.web import Application from tfw.networking import EventHandlerConnector +from tfw.config import TFWENV from tfw.config.logs import logging -from .zmq_websocket_proxy import ZMQWebSocketProxy +from .zmq_websocket_router import ZMQWebSocketRouter LOG = logging.getLogger(__name__) @@ -18,12 +19,15 @@ class TFWServer: SUB socket. """ def __init__(self): - self._event_handler_connector = EventHandlerConnector() + self._event_handler_connector = EventHandlerConnector( + downlink_bind_addr=f'tcp://*:{TFWENV.PULL_PORT}', + uplink_bind_addr=f'tcp://*:{TFWENV.PUB_PORT}' + ) self.application = Application([( - r'/ws', ZMQWebSocketProxy, { + r'/ws', ZMQWebSocketRouter, { 'event_handler_connector': self._event_handler_connector, } )]) - def listen(self, port): - self.application.listen(port) + def listen(self): + self.application.listen(TFWENV.WEB_PORT) diff --git a/lib/tfw/server/zmq_websocket_proxy.py b/lib/tfw/server/zmq_websocket_router.py similarity index 88% rename from lib/tfw/server/zmq_websocket_proxy.py rename to lib/tfw/server/zmq_websocket_router.py index b820f85..d6021b6 100644 --- a/lib/tfw/server/zmq_websocket_proxy.py +++ b/lib/tfw/server/zmq_websocket_router.py @@ -11,7 +11,7 @@ from tfw.config.logs import logging LOG = logging.getLogger(__name__) -class ZMQWebSocketProxy(WebSocketHandler): +class ZMQWebSocketRouter(WebSocketHandler): # pylint: disable=abstract-method instances = set() @@ -22,16 +22,16 @@ class ZMQWebSocketProxy(WebSocketHandler): def send_to_zmq(self, message): self.event_handler_connector.send_message(message) - @staticmethod - def send_to_websockets(message): - for instance in ZMQWebSocketProxy.instances: + @classmethod + def send_to_websockets(cls, message): + for instance in cls.instances: instance.write_message(message) def prepare(self): - ZMQWebSocketProxy.instances.add(self) + type(self).instances.add(self) def on_close(self): - ZMQWebSocketProxy.instances.remove(self) + type(self).instances.remove(self) def open(self, *args, **kwargs): LOG.debug('WebSocket connection initiated!') diff --git a/supervisor/tfw_server.py b/supervisor/tfw_server.py index 4cb8bd6..a3848b6 100644 --- a/supervisor/tfw_server.py +++ b/supervisor/tfw_server.py @@ -1,9 +1,8 @@ from tornado.ioloop import IOLoop from tfw.server import TFWServer -from tfw.config import TFWENV if __name__ == '__main__': - TFWServer().listen(TFWENV.WEB_PORT) + TFWServer().listen() IOLoop.instance().start() From dfac686bfc5ac377c630774e538978c75d6cf42a Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 31 May 2019 09:36:19 +0200 Subject: [PATCH 017/174] Create inotify module with unit tests --- lib/tfw/components/inotify/__init__.py | 6 + lib/tfw/components/inotify/inotify.py | 168 +++++++++++++++++++ lib/tfw/components/inotify/test_inotify.py | 185 +++++++++++++++++++++ 3 files changed, 359 insertions(+) create mode 100644 lib/tfw/components/inotify/__init__.py create mode 100644 lib/tfw/components/inotify/inotify.py create mode 100644 lib/tfw/components/inotify/test_inotify.py diff --git a/lib/tfw/components/inotify/__init__.py b/lib/tfw/components/inotify/__init__.py new file mode 100644 index 0000000..1b97c47 --- /dev/null +++ b/lib/tfw/components/inotify/__init__.py @@ -0,0 +1,6 @@ +from .inotify import InotifyObserver +from .inotify import ( + InotifyFileCreatedEvent, InotifyFileModifiedEvent, InotifyFileMovedEvent, + InotifyFileDeletedEvent, InotifyDirCreatedEvent, InotifyDirModifiedEvent, + InotifyDirMovedEvent, InotifyDirDeletedEvent +) diff --git a/lib/tfw/components/inotify/inotify.py b/lib/tfw/components/inotify/inotify.py new file mode 100644 index 0000000..f26ce6b --- /dev/null +++ b/lib/tfw/components/inotify/inotify.py @@ -0,0 +1,168 @@ +# pylint: disable=too-few-public-methods + +from time import time + +from watchdog.observers import Observer +from watchdog.events import FileSystemMovedEvent, PatternMatchingEventHandler +from watchdog.events import ( + FileCreatedEvent, FileModifiedEvent, FileMovedEvent, FileDeletedEvent, + DirCreatedEvent, DirModifiedEvent, DirMovedEvent, DirDeletedEvent +) + + +class InotifyEvent: + def __init__(self, src_path): + self.date = time() + self.src_path = src_path + + def __str__(self): + return self.__repr__() + + def __repr__(self): + return f'{self.date}\t{self.__class__.__name__}:\t{self.src_path}' + + +class InotifyMovedEvent(InotifyEvent): + def __init__(self, src_path, dest_path): + self.dest_path = dest_path + super().__init__(src_path) + + def __repr__(self): + return super().__repr__()+f'\t-> {self.dest_path}' + + +class InotifyFileCreatedEvent(InotifyEvent): + pass + + +class InotifyFileModifiedEvent(InotifyEvent): + pass + + +class InotifyFileMovedEvent(InotifyMovedEvent): + pass + + +class InotifyFileDeletedEvent(InotifyEvent): + pass + + +class InotifyDirCreatedEvent(InotifyEvent): + pass + + +class InotifyDirModifiedEvent(InotifyEvent): + pass + + +class InotifyDirMovedEvent(InotifyMovedEvent): + pass + + +class InotifyDirDeletedEvent(InotifyEvent): + pass + + +class InotifyObserver: + def __init__(self, path, patterns=None, exclude=None): + self._path = path + self._patterns = patterns + self._exclude = exclude + self.observer = Observer() + self.reset(path, patterns, exclude) + + def reset(self, path, patterns, exclude): + dispatch_event = self.dispatch_event + class TransformerEventHandler(PatternMatchingEventHandler): + def on_any_event(self, event): + dispatch_event(event) + self.handler = TransformerEventHandler(patterns, exclude) + self.observer.unschedule_all() + self.observer.schedule(self.handler, path, recursive=True) + + @property + def path(self): + return self._path + + @path.setter + def path(self, path): + self._path = path + self.reset(path, self._patterns, self._exclude) + + @property + def patterns(self): + return self._patterns + + @patterns.setter + def patterns(self, patterns): + self._patterns = patterns + self.reset(self._path, patterns, self._exclude) + + @property + def exclude(self): + return self._exclude + + @exclude.setter + def exclude(self, exclude): + self._exclude = exclude + self.reset(self._path, self._patterns, exclude) + + def start(self): + self.observer.start() + + def stop(self): + self.observer.stop() + self.observer.join() + + def dispatch_event(self, event): + event_to_action = { + InotifyFileCreatedEvent : self.on_created, + InotifyFileModifiedEvent : self.on_modified, + InotifyFileMovedEvent : self.on_moved, + InotifyFileDeletedEvent : self.on_deleted, + InotifyDirCreatedEvent : self.on_created, + InotifyDirModifiedEvent : self.on_modified, + InotifyDirMovedEvent : self.on_moved, + InotifyDirDeletedEvent : self.on_deleted + } + + event = self.transform_event(event) + self.on_any_event(event) + event_to_action[type(event)](event) + + @staticmethod + def transform_event(event): + watchdog_to_inotify = { + FileCreatedEvent : InotifyFileCreatedEvent, + FileModifiedEvent : InotifyFileModifiedEvent, + FileMovedEvent : InotifyFileMovedEvent, + FileDeletedEvent : InotifyFileDeletedEvent, + DirCreatedEvent : InotifyDirCreatedEvent, + DirModifiedEvent : InotifyDirModifiedEvent, + DirMovedEvent : InotifyDirMovedEvent, + DirDeletedEvent : InotifyDirDeletedEvent + } + + try: + cls = watchdog_to_inotify[type(event)] + except KeyError: + raise NameError('Watchdog API returned an unknown event.') + + if isinstance(event, FileSystemMovedEvent): + return cls(event.src_path, event.dest_path) + return cls(event.src_path) + + def on_any_event(self, event): + pass + + def on_created(self, event): + pass + + def on_modified(self, event): + pass + + def on_moved(self, event): + pass + + def on_deleted(self, event): + pass diff --git a/lib/tfw/components/inotify/test_inotify.py b/lib/tfw/components/inotify/test_inotify.py new file mode 100644 index 0000000..1503a34 --- /dev/null +++ b/lib/tfw/components/inotify/test_inotify.py @@ -0,0 +1,185 @@ +# pylint: disable=redefined-outer-name + +from queue import Empty, Queue +from secrets import token_urlsafe as random_name +from pathlib import Path +from shutil import rmtree +from os.path import join +from os import chdir, mkdir, rename, walk, listdir +from tempfile import TemporaryDirectory + +import pytest + +from inotify import InotifyObserver +from inotify import ( + InotifyFileCreatedEvent, InotifyFileModifiedEvent, InotifyFileMovedEvent, + InotifyFileDeletedEvent, InotifyDirCreatedEvent, InotifyDirModifiedEvent, + InotifyDirMovedEvent, InotifyDirDeletedEvent +) + + +class InotifyContext: + def __init__(self, folder, observer): + self.missing = 0 + self.folder = folder + self.observer = observer + + self.event_to_queue = { + InotifyFileCreatedEvent : self.observer.create_queue, + InotifyFileModifiedEvent : self.observer.modify_queue, + InotifyFileMovedEvent : self.observer.move_queue, + InotifyFileDeletedEvent : self.observer.delete_queue, + InotifyDirCreatedEvent : self.observer.create_queue, + InotifyDirModifiedEvent : self.observer.modify_queue, + InotifyDirMovedEvent : self.observer.move_queue, + InotifyDirDeletedEvent : self.observer.delete_queue + } + + def random_file(self, dirname, extension): + filename = self.join(f'{dirname}/{random_name(16)}{extension}') + Path(filename).touch() + return filename + + def random_folder(self, basepath): + dirname = self.join(f'{basepath}/{random_name(16)}') + mkdir(dirname) + return dirname + + def join(self, path): + return join(self.folder, path) + + def check_event(self, event_type, path): + event = self.event_to_queue[event_type].get(timeout=1) + assert isinstance(event, event_type) + assert event.src_path == path + return event + + def check_empty(self, event_type): + with pytest.raises(Empty): + self.event_to_queue[event_type].get(timeout=1) + + def check_any(self, missing): + self.missing += missing + attrs = self.observer.__dict__.values() + total = sum([q.qsize() for q in attrs if isinstance(q, Queue)]) + return total+self.missing == len(self.observer.any_list) + + +class InotifyTestObserver(InotifyObserver): + def __init__(self, path, patterns=None, exclude=None): + self.any_list = [] + (self.create_queue, self.modify_queue, self.move_queue, + self.delete_queue) = [Queue() for _ in range(4)] + super().__init__(path, patterns, exclude) + + def on_any_event(self, event): + self.any_list.append(event) + + def on_created(self, event): + self.create_queue.put(event) + + def on_modified(self, event): + self.modify_queue.put(event) + + def on_moved(self, event): + self.move_queue.put(event) + + def on_deleted(self, event): + self.delete_queue.put(event) + + +@pytest.fixture() +def context(): + with TemporaryDirectory() as workdir: + chdir(workdir) + + for _ in range(5): + newdir = join(workdir, random_name(16)) + mkdir(newdir) + Path(join(newdir, random_name(16)+'.txt')).touch() + + monitor = InotifyTestObserver(workdir) + monitor.start() + yield InotifyContext(workdir, monitor) + +def test_create(context): + for _, dirs, _ in list(walk(context.folder)): + for name in dirs: + newfile = context.random_file(name, '.txt') + context.check_event(InotifyFileCreatedEvent, newfile) + newdir = context.random_folder(name) + context.check_event(InotifyDirCreatedEvent, newdir) + assert context.check_any(2) + +def test_modify(context): + for root, _, files in list(walk(context.folder)): + for name in files: + oldfile = join(root, name) + with open(oldfile, 'w') as ofile: + ofile.write('text') + context.check_event(InotifyFileModifiedEvent, oldfile) + rename(oldfile, oldfile+'_new') + context.check_event(InotifyDirModifiedEvent, root) + assert context.check_any(2) + +def test_move(context): + for root, dirs, _ in list(walk(context.folder)): + for name in dirs: + olddir = join(root, name) + oldfile = join(olddir, list(listdir(olddir))[0]) + rename(olddir, olddir+'_new') + context.check_event(InotifyDirMovedEvent, olddir) + context.check_event(InotifyFileMovedEvent, oldfile) + assert context.check_any(2) + +def test_delete(context): + for root, dirs, _ in list(walk(context.folder)): + for name in dirs: + olddir = join(root, name) + oldfile = join(olddir, list(listdir(olddir))[0]) + rmtree(olddir) + context.check_event(InotifyFileDeletedEvent, oldfile) + context.check_event(InotifyDirDeletedEvent, olddir) + assert context.check_any(2) + +def test_path(context): + for _, dirs, _ in list(walk(context.folder)): + for name in dirs: + context.observer.path = context.join(name) + context.random_folder('.') + newfile = context.random_file(name, '.txt') + context.check_event(InotifyFileCreatedEvent, newfile) + context.observer.path = context.folder + assert context.check_any(1) + +def test_patterns(context): + for _, dirs, _ in list(walk(context.folder)): + for name in dirs: + context.observer.patterns = ["*.txt"] + context.random_file(name, '.bin') + newfile = context.random_file(name, '.txt') + context.check_event(InotifyFileCreatedEvent, newfile) + context.check_empty(InotifyFileCreatedEvent) + assert context.check_any(1) + context.observer.patterns = None + +def test_exclude(context): + for _, dirs, _ in list(walk(context.folder)): + for name in dirs: + context.observer.exclude = ["*.txt"] + context.random_file(name, '.txt') + newfile = context.random_file(name, '.bin') + context.check_event(InotifyFileCreatedEvent, newfile) + context.check_empty(InotifyFileCreatedEvent) + assert context.check_any(1) + context.observer.exclude = None + +def test_stress(context): + for _, dirs, _ in list(walk(context.folder)): + for name in dirs: + newfile = [] + for i in range(1024): + newfile.append(context.random_file(name, '.txt')) + for i in range(1024): + context.check_event(InotifyFileCreatedEvent, newfile[i]) + assert context.check_any(1024) From ca6797b3544c1fe72f2c002b76cb743c7da689fa Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 6 Jun 2019 13:41:13 +0200 Subject: [PATCH 018/174] Fixing issues in PR #56 --- lib/tfw/components/inotify/test_inotify.py | 61 +++++++++++----------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/lib/tfw/components/inotify/test_inotify.py b/lib/tfw/components/inotify/test_inotify.py index 1503a34..702e2c7 100644 --- a/lib/tfw/components/inotify/test_inotify.py +++ b/lib/tfw/components/inotify/test_inotify.py @@ -1,7 +1,7 @@ # pylint: disable=redefined-outer-name from queue import Empty, Queue -from secrets import token_urlsafe as random_name +from secrets import token_urlsafe from pathlib import Path from shutil import rmtree from os.path import join @@ -20,7 +20,7 @@ from inotify import ( class InotifyContext: def __init__(self, folder, observer): - self.missing = 0 + self.missing_events = 0 self.folder = folder self.observer = observer @@ -35,13 +35,13 @@ class InotifyContext: InotifyDirDeletedEvent : self.observer.delete_queue } - def random_file(self, dirname, extension): - filename = self.join(f'{dirname}/{random_name(16)}{extension}') + def create_random_file(self, dirname, extension): + filename = self.join(f'{dirname}/{generate_name()}{extension}') Path(filename).touch() return filename - def random_folder(self, basepath): - dirname = self.join(f'{basepath}/{random_name(16)}') + def create_random_folder(self, basepath): + dirname = self.join(f'{basepath}/{generate_name()}') mkdir(dirname) return dirname @@ -49,6 +49,7 @@ class InotifyContext: return join(self.folder, path) def check_event(self, event_type, path): + self.missing_events += 1 event = self.event_to_queue[event_type].get(timeout=1) assert isinstance(event, event_type) assert event.src_path == path @@ -58,18 +59,16 @@ class InotifyContext: with pytest.raises(Empty): self.event_to_queue[event_type].get(timeout=1) - def check_any(self, missing): - self.missing += missing + def check_any(self): attrs = self.observer.__dict__.values() total = sum([q.qsize() for q in attrs if isinstance(q, Queue)]) - return total+self.missing == len(self.observer.any_list) + return total+self.missing_events == len(self.observer.any_list) class InotifyTestObserver(InotifyObserver): def __init__(self, path, patterns=None, exclude=None): self.any_list = [] - (self.create_queue, self.modify_queue, self.move_queue, - self.delete_queue) = [Queue() for _ in range(4)] + self.create_queue, self.modify_queue, self.move_queue, self.delete_queue = [Queue() for _ in range(4)] super().__init__(path, patterns, exclude) def on_any_event(self, event): @@ -87,6 +86,8 @@ class InotifyTestObserver(InotifyObserver): def on_deleted(self, event): self.delete_queue.put(event) +def generate_name(): + return token_urlsafe(16) @pytest.fixture() def context(): @@ -94,9 +95,9 @@ def context(): chdir(workdir) for _ in range(5): - newdir = join(workdir, random_name(16)) + newdir = join(workdir, generate_name()) mkdir(newdir) - Path(join(newdir, random_name(16)+'.txt')).touch() + Path(join(newdir, generate_name()+'.txt')).touch() monitor = InotifyTestObserver(workdir) monitor.start() @@ -105,11 +106,11 @@ def context(): def test_create(context): for _, dirs, _ in list(walk(context.folder)): for name in dirs: - newfile = context.random_file(name, '.txt') + newfile = context.create_random_file(name, '.txt') context.check_event(InotifyFileCreatedEvent, newfile) - newdir = context.random_folder(name) + newdir = context.create_random_folder(name) context.check_event(InotifyDirCreatedEvent, newdir) - assert context.check_any(2) + assert context.check_any() def test_modify(context): for root, _, files in list(walk(context.folder)): @@ -120,7 +121,7 @@ def test_modify(context): context.check_event(InotifyFileModifiedEvent, oldfile) rename(oldfile, oldfile+'_new') context.check_event(InotifyDirModifiedEvent, root) - assert context.check_any(2) + assert context.check_any() def test_move(context): for root, dirs, _ in list(walk(context.folder)): @@ -130,7 +131,7 @@ def test_move(context): rename(olddir, olddir+'_new') context.check_event(InotifyDirMovedEvent, olddir) context.check_event(InotifyFileMovedEvent, oldfile) - assert context.check_any(2) + assert context.check_any() def test_delete(context): for root, dirs, _ in list(walk(context.folder)): @@ -140,38 +141,38 @@ def test_delete(context): rmtree(olddir) context.check_event(InotifyFileDeletedEvent, oldfile) context.check_event(InotifyDirDeletedEvent, olddir) - assert context.check_any(2) + assert context.check_any() def test_path(context): for _, dirs, _ in list(walk(context.folder)): for name in dirs: context.observer.path = context.join(name) - context.random_folder('.') - newfile = context.random_file(name, '.txt') + context.create_random_folder('.') + newfile = context.create_random_file(name, '.txt') context.check_event(InotifyFileCreatedEvent, newfile) context.observer.path = context.folder - assert context.check_any(1) + assert context.check_any() def test_patterns(context): for _, dirs, _ in list(walk(context.folder)): for name in dirs: context.observer.patterns = ["*.txt"] - context.random_file(name, '.bin') - newfile = context.random_file(name, '.txt') + context.create_random_file(name, '.bin') + newfile = context.create_random_file(name, '.txt') context.check_event(InotifyFileCreatedEvent, newfile) context.check_empty(InotifyFileCreatedEvent) - assert context.check_any(1) + assert context.check_any() context.observer.patterns = None def test_exclude(context): for _, dirs, _ in list(walk(context.folder)): for name in dirs: context.observer.exclude = ["*.txt"] - context.random_file(name, '.txt') - newfile = context.random_file(name, '.bin') + context.create_random_file(name, '.txt') + newfile = context.create_random_file(name, '.bin') context.check_event(InotifyFileCreatedEvent, newfile) context.check_empty(InotifyFileCreatedEvent) - assert context.check_any(1) + assert context.check_any() context.observer.exclude = None def test_stress(context): @@ -179,7 +180,7 @@ def test_stress(context): for name in dirs: newfile = [] for i in range(1024): - newfile.append(context.random_file(name, '.txt')) + newfile.append(context.create_random_file(name, '.txt')) for i in range(1024): context.check_event(InotifyFileCreatedEvent, newfile[i]) - assert context.check_any(1024) + assert context.check_any() From 3d00cbeebc43b9afd7f1aaa0be97e256faa91d93 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 6 Jun 2019 16:57:15 +0200 Subject: [PATCH 019/174] Add SQLAlchemy as dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 1b5f83c..610702d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ PyYAML>=5.0.0,<6.0.0 Jinja2>=2.0.0,<3.0.0 cryptography>=2.0.0,<3.0.0 python-dateutil>=2.0.0,<3.0.0 +SQLAlchemy==1.3.4 From 105a574d7fdf5584f304c0200786a2d50741bc43 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 10 Jun 2019 15:32:45 +0200 Subject: [PATCH 020/174] Import the logging modules directly --- lib/tfw/components/directory_monitor.py | 3 +-- lib/tfw/components/directory_monitoring_event_handler.py | 2 +- lib/tfw/components/directory_snapshotting_event_handler.py | 2 +- lib/tfw/components/fsm_managing_event_handler.py | 3 ++- lib/tfw/components/ide_event_handler.py | 2 +- lib/tfw/components/log_monitoring_event_handler.py | 3 ++- lib/tfw/components/pipe_io_event_handler.py | 2 +- lib/tfw/components/process_managing_event_handler.py | 2 +- lib/tfw/components/terminado_mini_server.py | 3 ++- lib/tfw/components/terminal_commands.py | 3 +-- lib/tfw/components/terminal_event_handler.py | 3 ++- lib/tfw/config/logs.py | 6 ------ lib/tfw/event_handlers/event_handler_base.py | 3 +-- lib/tfw/event_handlers/fsm_aware.py | 3 ++- lib/tfw/fsm/fsm_base.py | 2 +- lib/tfw/mixins/monitor_manager_mixin.py | 2 +- lib/tfw/networking/event_handler_connector.py | 4 ++-- lib/tfw/networking/server_connector.py | 3 +-- lib/tfw/server/tfw_server.py | 3 ++- lib/tfw/server/zmq_websocket_router.py | 3 ++- supervisor/tfw_server.py | 4 ++++ 21 files changed, 31 insertions(+), 30 deletions(-) delete mode 100644 lib/tfw/config/logs.py diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index 7144f0e..65dc275 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -1,6 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging from functools import wraps from watchdog.events import FileSystemEventHandler as FileSystemWatchdogEventHandler @@ -10,8 +11,6 @@ from tfw.event_handlers import TFWServerUplinkConnector from tfw.decorators.rate_limiter import RateLimiter from tfw.mixins.observer_mixin import ObserverMixin -from tfw.config.logs import logging - LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/directory_monitoring_event_handler.py b/lib/tfw/components/directory_monitoring_event_handler.py index 18bb8ca..222736c 100644 --- a/lib/tfw/components/directory_monitoring_event_handler.py +++ b/lib/tfw/components/directory_monitoring_event_handler.py @@ -1,12 +1,12 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging from os.path import isdir, exists from tfw.event_handlers import FrontendEventHandlerBase from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin from tfw.components.directory_monitor import DirectoryMonitor -from tfw.config.logs import logging LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/directory_snapshotting_event_handler.py b/lib/tfw/components/directory_snapshotting_event_handler.py index d60732f..1e8c1ea 100644 --- a/lib/tfw/components/directory_snapshotting_event_handler.py +++ b/lib/tfw/components/directory_snapshotting_event_handler.py @@ -1,6 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging from os.path import join as joinpath from os.path import basename from os import makedirs @@ -11,7 +12,6 @@ from dateutil import parser as dateparser from tfw.event_handlers import FrontendEventHandlerBase from tfw.components.snapshot_provider import SnapshotProvider from tfw.config import TFWENV -from tfw.config.logs import logging LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/fsm_managing_event_handler.py b/lib/tfw/components/fsm_managing_event_handler.py index ac6e6fc..4860f5a 100644 --- a/lib/tfw/components/fsm_managing_event_handler.py +++ b/lib/tfw/components/fsm_managing_event_handler.py @@ -1,9 +1,10 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging + from tfw.event_handlers import FrontendEventHandlerBase from tfw.crypto import KeyManager, sign_message, verify_message -from tfw.config.logs import logging from tfw.networking import Scope LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/ide_event_handler.py b/lib/tfw/components/ide_event_handler.py index 2632b45..da32cda 100644 --- a/lib/tfw/components/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler.py @@ -1,6 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging from os.path import isfile, join, relpath, exists, isdir, realpath from glob import glob from fnmatch import fnmatchcase @@ -9,7 +10,6 @@ from typing import Iterable from tfw.event_handlers import FrontendEventHandlerBase from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin from tfw.components.directory_monitor import DirectoryMonitor -from tfw.config.logs import logging LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/log_monitoring_event_handler.py b/lib/tfw/components/log_monitoring_event_handler.py index 23fe5b9..e324786 100644 --- a/lib/tfw/components/log_monitoring_event_handler.py +++ b/lib/tfw/components/log_monitoring_event_handler.py @@ -1,10 +1,11 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging + from tfw.event_handlers import FrontendEventHandlerBase from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin from tfw.components.log_monitor import LogMonitor -from tfw.config.logs import logging LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/pipe_io_event_handler.py b/lib/tfw/components/pipe_io_event_handler.py index e35455e..da3e726 100644 --- a/lib/tfw/components/pipe_io_event_handler.py +++ b/lib/tfw/components/pipe_io_event_handler.py @@ -1,3 +1,4 @@ +import logging from abc import abstractmethod from json import loads, dumps from subprocess import run, PIPE, Popen @@ -10,7 +11,6 @@ from threading import Thread from contextlib import suppress from tfw.event_handlers import EventHandlerBase -from tfw.config.logs import logging from .pipe_io_server import PipeIOServer, terminate_process_on_failure diff --git a/lib/tfw/components/process_managing_event_handler.py b/lib/tfw/components/process_managing_event_handler.py index f6489e6..37141d0 100644 --- a/lib/tfw/components/process_managing_event_handler.py +++ b/lib/tfw/components/process_managing_event_handler.py @@ -1,12 +1,12 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging from xmlrpc.client import Fault as SupervisorFault from tfw.event_handlers import FrontendEventHandlerBase from tfw.mixins.supervisor_mixin import SupervisorMixin, SupervisorLogMixin from tfw.components.directory_monitor import with_monitor_paused -from tfw.config.logs import logging LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/terminado_mini_server.py b/lib/tfw/components/terminado_mini_server.py index 0f77c34..a440e23 100644 --- a/lib/tfw/components/terminado_mini_server.py +++ b/lib/tfw/components/terminado_mini_server.py @@ -1,12 +1,13 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging + from tornado.ioloop import IOLoop from tornado.web import Application from terminado import TermSocket, SingleTermManager from tfw.config import TFWENV -from tfw.config.logs import logging LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/terminal_commands.py b/lib/tfw/components/terminal_commands.py index 14563bd..9a55d22 100644 --- a/lib/tfw/components/terminal_commands.py +++ b/lib/tfw/components/terminal_commands.py @@ -1,12 +1,11 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging from abc import ABC from re import match from shlex import split -from tfw.config.logs import logging - LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/components/terminal_event_handler.py index 3427c4f..436247f 100644 --- a/lib/tfw/components/terminal_event_handler.py +++ b/lib/tfw/components/terminal_event_handler.py @@ -1,10 +1,11 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging + from tfw.event_handlers import FrontendEventHandlerBase from tfw.components.terminado_mini_server import TerminadoMiniServer from tfw.config import TFWENV -from tfw.config.logs import logging from tao.config import TAOENV LOG = logging.getLogger(__name__) diff --git a/lib/tfw/config/logs.py b/lib/tfw/config/logs.py deleted file mode 100644 index 15992b2..0000000 --- a/lib/tfw/config/logs.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -import logging - -logging.basicConfig(level=logging.DEBUG) diff --git a/lib/tfw/event_handlers/event_handler_base.py b/lib/tfw/event_handlers/event_handler_base.py index 0cc0a9e..d5bb20e 100644 --- a/lib/tfw/event_handlers/event_handler_base.py +++ b/lib/tfw/event_handlers/event_handler_base.py @@ -1,12 +1,11 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging from abc import ABC, abstractmethod from inspect import currentframe from typing import Iterable -from tfw.config.logs import logging - from .tfw_server_connector import TFWServerConnector LOG = logging.getLogger(__name__) diff --git a/lib/tfw/event_handlers/fsm_aware.py b/lib/tfw/event_handlers/fsm_aware.py index 13cea43..ddbf6fb 100644 --- a/lib/tfw/event_handlers/fsm_aware.py +++ b/lib/tfw/event_handlers/fsm_aware.py @@ -1,8 +1,9 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging + from tfw.crypto import KeyManager, verify_message -from tfw.config.logs import logging LOG = logging.getLogger(__name__) diff --git a/lib/tfw/fsm/fsm_base.py b/lib/tfw/fsm/fsm_base.py index d9fff29..5b050e0 100644 --- a/lib/tfw/fsm/fsm_base.py +++ b/lib/tfw/fsm/fsm_base.py @@ -1,13 +1,13 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging from collections import defaultdict from datetime import datetime from transitions import Machine, MachineError from tfw.mixins.callback_mixin import CallbackMixin -from tfw.config.logs import logging LOG = logging.getLogger(__name__) diff --git a/lib/tfw/mixins/monitor_manager_mixin.py b/lib/tfw/mixins/monitor_manager_mixin.py index f247659..c153ff9 100644 --- a/lib/tfw/mixins/monitor_manager_mixin.py +++ b/lib/tfw/mixins/monitor_manager_mixin.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw.config.logs import logging +import logging LOG = logging.getLogger(__name__) diff --git a/lib/tfw/networking/event_handler_connector.py b/lib/tfw/networking/event_handler_connector.py index 378db11..e9b2511 100644 --- a/lib/tfw/networking/event_handler_connector.py +++ b/lib/tfw/networking/event_handler_connector.py @@ -1,11 +1,11 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging + import zmq from zmq.eventloop.zmqstream import ZMQStream -from tfw.config.logs import logging - from .serialization import serialize_tfw_msg, with_deserialize_tfw_msg LOG = logging.getLogger(__name__) diff --git a/lib/tfw/networking/server_connector.py b/lib/tfw/networking/server_connector.py index 6bee301..dc3c78c 100644 --- a/lib/tfw/networking/server_connector.py +++ b/lib/tfw/networking/server_connector.py @@ -1,13 +1,12 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging from functools import partial import zmq from zmq.eventloop.zmqstream import ZMQStream -from tfw.config.logs import logging - from .scope import Scope from .serialization import serialize_tfw_msg, with_deserialize_tfw_msg diff --git a/lib/tfw/server/tfw_server.py b/lib/tfw/server/tfw_server.py index ad2d37e..a7f7804 100644 --- a/lib/tfw/server/tfw_server.py +++ b/lib/tfw/server/tfw_server.py @@ -1,11 +1,12 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import logging + from tornado.web import Application from tfw.networking import EventHandlerConnector from tfw.config import TFWENV -from tfw.config.logs import logging from .zmq_websocket_router import ZMQWebSocketRouter diff --git a/lib/tfw/server/zmq_websocket_router.py b/lib/tfw/server/zmq_websocket_router.py index d6021b6..43e46f4 100644 --- a/lib/tfw/server/zmq_websocket_router.py +++ b/lib/tfw/server/zmq_websocket_router.py @@ -2,13 +2,14 @@ # All Rights Reserved. See LICENSE file for details. import json +import logging from tornado.websocket import WebSocketHandler from tfw.networking import Scope -from tfw.config.logs import logging LOG = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) class ZMQWebSocketRouter(WebSocketHandler): diff --git a/supervisor/tfw_server.py b/supervisor/tfw_server.py index a3848b6..0d52bbf 100644 --- a/supervisor/tfw_server.py +++ b/supervisor/tfw_server.py @@ -1,7 +1,11 @@ +import logging + from tornado.ioloop import IOLoop from tfw.server import TFWServer +logging.basicConfig(level=logging.DEBUG) + if __name__ == '__main__': TFWServer().listen() From 8be0105ef68e3a9096221de1fc2aaa137d1d0858 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 11 Jun 2019 11:07:12 +0200 Subject: [PATCH 021/174] Remove unnecessary config --- lib/tfw/server/zmq_websocket_router.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/tfw/server/zmq_websocket_router.py b/lib/tfw/server/zmq_websocket_router.py index 43e46f4..8ad0905 100644 --- a/lib/tfw/server/zmq_websocket_router.py +++ b/lib/tfw/server/zmq_websocket_router.py @@ -9,7 +9,6 @@ from tornado.websocket import WebSocketHandler from tfw.networking import Scope LOG = logging.getLogger(__name__) -logging.basicConfig(level=logging.DEBUG) class ZMQWebSocketRouter(WebSocketHandler): From a69031015b4b5a2f451a1d5cf24a4dda7196364f Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 20 May 2019 14:50:02 +0200 Subject: [PATCH 022/174] Refactor file manager --- .../components/{ => ide_event_handler}/ide_event_handler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) rename lib/tfw/components/{ => ide_event_handler}/ide_event_handler.py (98%) diff --git a/lib/tfw/components/ide_event_handler.py b/lib/tfw/components/ide_event_handler/ide_event_handler.py similarity index 98% rename from lib/tfw/components/ide_event_handler.py rename to lib/tfw/components/ide_event_handler/ide_event_handler.py index da32cda..19d8652 100644 --- a/lib/tfw/components/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler/ide_event_handler.py @@ -16,7 +16,7 @@ LOG = logging.getLogger(__name__) class FileManager: # pylint: disable=too-many-instance-attributes def __init__(self, working_directory, allowed_directories, selected_file=None, exclude=None): - self._exclude, self.exclude = None, exclude + self._exclude, self.exclude = [], exclude self._allowed_directories, self.allowed_directories = None, allowed_directories self._workdir, self.workdir = None, working_directory self._filename, self.filename = None, selected_file or self.files[0] @@ -91,7 +91,8 @@ class FileManager: # pylint: disable=too-many-instance-attributes def _is_blacklisted(self, file): return any( - fnmatchcase(file, blacklisted) + fnmatchcase(file, blacklisted) or + fnmatchcase(basename(file), blacklisted) for blacklisted in self.exclude ) From 9cb8ef0e7276051d73b7b8bc65e4a65026d318b6 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 20 May 2019 14:52:02 +0200 Subject: [PATCH 023/174] Add unit test for file manager --- .../ide_event_handler/file_manager.py | 93 ++++++++++++++++++ .../ide_event_handler/ide_event_handler.py | 91 +---------------- .../ide_event_handler/test_filemanager.py | 97 +++++++++++++++++++ 3 files changed, 192 insertions(+), 89 deletions(-) create mode 100644 lib/tfw/components/ide_event_handler/file_manager.py create mode 100644 lib/tfw/components/ide_event_handler/test_filemanager.py diff --git a/lib/tfw/components/ide_event_handler/file_manager.py b/lib/tfw/components/ide_event_handler/file_manager.py new file mode 100644 index 0000000..59431ea --- /dev/null +++ b/lib/tfw/components/ide_event_handler/file_manager.py @@ -0,0 +1,93 @@ +from typing import Iterable +from glob import glob +from fnmatch import fnmatchcase +from os.path import basename, isfile, join, relpath, exists, isdir, realpath + + +class FileManager: # pylint: disable=too-many-instance-attributes + def __init__(self, working_directory, allowed_directories, selected_file=None, exclude=None): + self._exclude, self.exclude = [], exclude + self._allowed_directories, self.allowed_directories = None, allowed_directories + self._workdir, self.workdir = None, working_directory + self._filename, self.filename = None, selected_file or self.files[0] + + @property + def exclude(self): + return self._exclude + + @exclude.setter + def exclude(self, exclude): + if exclude is None: + return + if not isinstance(exclude, Iterable): + raise TypeError('Exclude must be Iterable!') + self._exclude = exclude + + @property + def workdir(self): + return self._workdir + + @workdir.setter + def workdir(self, directory): + if not exists(directory) or not isdir(directory): + raise EnvironmentError(f'"{directory}" is not a directory!') + if not self._is_in_allowed_dir(directory): + raise EnvironmentError(f'Directory "{directory}" is not allowed!') + self._workdir = directory + + @property + def allowed_directories(self): + return self._allowed_directories + + @allowed_directories.setter + def allowed_directories(self, directories): + self._allowed_directories = directories + + @property + def filename(self): + return self._filename + + @filename.setter + def filename(self, filename): + if filename not in self.files: + raise EnvironmentError('No such file in workdir!') + self._filename = filename + + @property + def files(self): + return [ + self._relpath(file) + for file in glob(join(self._workdir, '**/*'), recursive=True) + if isfile(file) + and self._is_in_allowed_dir(file) + and not self._is_blacklisted(file) + ] + + @property + def file_contents(self): + with open(self._filepath(self.filename), 'r', errors='surrogateescape') as ifile: + return ifile.read() + + @file_contents.setter + def file_contents(self, value): + with open(self._filepath(self.filename), 'w', errors='surrogateescape') as ofile: + ofile.write(value) + + def _is_in_allowed_dir(self, path): + return any( + realpath(path).startswith(allowed_dir) + for allowed_dir in self.allowed_directories + ) + + def _is_blacklisted(self, file): + return any( + fnmatchcase(file, blacklisted) or + fnmatchcase(basename(file), blacklisted) + for blacklisted in self.exclude + ) + + def _filepath(self, filename): + return join(self._workdir, filename) + + def _relpath(self, filename): + return relpath(self._filepath(filename), start=self._workdir) diff --git a/lib/tfw/components/ide_event_handler/ide_event_handler.py b/lib/tfw/components/ide_event_handler/ide_event_handler.py index 19d8652..218d3ed 100644 --- a/lib/tfw/components/ide_event_handler/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler/ide_event_handler.py @@ -11,98 +11,11 @@ from tfw.event_handlers import FrontendEventHandlerBase from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin from tfw.components.directory_monitor import DirectoryMonitor +from .file_manager import FileManager + LOG = logging.getLogger(__name__) -class FileManager: # pylint: disable=too-many-instance-attributes - def __init__(self, working_directory, allowed_directories, selected_file=None, exclude=None): - self._exclude, self.exclude = [], exclude - self._allowed_directories, self.allowed_directories = None, allowed_directories - self._workdir, self.workdir = None, working_directory - self._filename, self.filename = None, selected_file or self.files[0] - - @property - def exclude(self): - return self._exclude - - @exclude.setter - def exclude(self, exclude): - if exclude is None: - return - if not isinstance(exclude, Iterable): - raise TypeError('Exclude must be Iterable!') - self._exclude = exclude - - @property - def workdir(self): - return self._workdir - - @workdir.setter - def workdir(self, directory): - if not exists(directory) or not isdir(directory): - raise EnvironmentError(f'"{directory}" is not a directory!') - if not self._is_in_allowed_dir(directory): - raise EnvironmentError(f'Directory "{directory}" is not allowed!') - self._workdir = directory - - @property - def allowed_directories(self): - return self._allowed_directories - - @allowed_directories.setter - def allowed_directories(self, directories): - self._allowed_directories = directories - - @property - def filename(self): - return self._filename - - @filename.setter - def filename(self, filename): - if filename not in self.files: - raise EnvironmentError('No such file in workdir!') - self._filename = filename - - @property - def files(self): - return [ - self._relpath(file) - for file in glob(join(self._workdir, '**/*'), recursive=True) - if isfile(file) - and self._is_in_allowed_dir(file) - and not self._is_blacklisted(file) - ] - - @property - def file_contents(self): - with open(self._filepath(self.filename), 'r', errors='surrogateescape') as ifile: - return ifile.read() - - @file_contents.setter - def file_contents(self, value): - with open(self._filepath(self.filename), 'w', errors='surrogateescape') as ofile: - ofile.write(value) - - def _is_in_allowed_dir(self, path): - return any( - realpath(path).startswith(allowed_dir) - for allowed_dir in self.allowed_directories - ) - - def _is_blacklisted(self, file): - return any( - fnmatchcase(file, blacklisted) or - fnmatchcase(basename(file), blacklisted) - for blacklisted in self.exclude - ) - - def _filepath(self, filename): - return join(self._workdir, filename) - - def _relpath(self, filename): - return relpath(self._filepath(filename), start=self._workdir) - - class IdeEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): # pylint: disable=too-many-arguments,anomalous-backslash-in-string """ diff --git a/lib/tfw/components/ide_event_handler/test_filemanager.py b/lib/tfw/components/ide_event_handler/test_filemanager.py new file mode 100644 index 0000000..d62f3e0 --- /dev/null +++ b/lib/tfw/components/ide_event_handler/test_filemanager.py @@ -0,0 +1,97 @@ +# pylint: disable=redefined-outer-name + +from secrets import token_urlsafe +from pathlib import Path +from shutil import rmtree +from os.path import join, realpath +from os import mkdir, rmdir, remove, symlink + +import pytest + +from filemanager import FileManager + + +WORKDIR = realpath('test_filemanager') + +def workdir_pref(path): + return join(WORKDIR, path) + +@pytest.fixture(scope='module') +def manager(): + dirs = [] + mkdir(WORKDIR) + + for i in range(3): + node = workdir_pref('dir_'+str(i).zfill(2)) + mkdir(node) + Path(join(node, 'empty.txt')).touch() + Path(join(node, 'empty.bin')).touch() + dirs.append(node) + + yield FileManager(dirs[0], dirs[:-1], exclude=['*/dir_01/*', '*.bin']) + rmtree(WORKDIR) + +@pytest.mark.parametrize('subdir', ['dir_00', 'dir_01']) +def test_select_allowed_dirs(manager, subdir): + manager.workdir = workdir_pref(subdir) + assert manager.workdir == workdir_pref(subdir) + newdir = workdir_pref(join(subdir, 'deep')) + mkdir(newdir) + manager.workdir = newdir + assert manager.workdir == newdir + rmdir(newdir) + +@pytest.mark.parametrize('excdir', ['/', workdir_pref('dir_02')]) +def test_select_excluded_dirs(manager, excdir): + allowed = manager.allowed_directories + with pytest.raises(OSError): + manager.workdir = excdir + assert manager.workdir != excdir + manager.allowed_directories = allowed+[excdir] + manager.workdir = excdir + assert manager.workdir == excdir + manager.allowed_directories = allowed + +@pytest.mark.parametrize('filename', ['another.txt', '*.txt']) +def test_select_allowed_files(manager, filename): + manager.workdir = workdir_pref('dir_00') + newfile = workdir_pref(join('dir_00', filename)) + Path(newfile).touch() + assert filename in manager.files + manager.filename = filename + assert manager.filename == filename + remove(newfile) + +@pytest.mark.parametrize('path', [ + ['dir_00', 'illegal.bin'], + ['dir_01', 'legal.txt'] +]) +def test_select_excluded_files(manager, path): + manager.workdir = workdir_pref(path[0]) + newfile = workdir_pref(join(path[0], path[1])) + Path(newfile).touch() + assert path[1] not in manager.files + with pytest.raises(OSError): + manager.filename = path[1] + remove(newfile) + +@pytest.mark.parametrize('path', [ + ['dir_02/empty.txt', 'dir_00/link.txt'], + ['dir_01/empty.txt', 'dir_00/link.bin'] +]) +def test_select_excluded_symlinks(manager, path): + manager.workdir = workdir_pref('dir_00') + link = workdir_pref(path[1]) + symlink(workdir_pref(path[0]), link) + assert path[1] not in manager.files + remove(link) + +def test_read_write_file(manager): + for _ in range(128): + manager.workdir = workdir_pref('dir_00') + manager.filename = 'empty.txt' + content = token_urlsafe(32) + manager.file_contents = content + assert manager.file_contents == content + with open(workdir_pref('dir_00/empty.txt'), "r") as ifile: + assert ifile.read() == content From b44fd200c657c7c941df71c6e4e722796ac96b4c Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 22 May 2019 22:31:55 +0200 Subject: [PATCH 024/174] Make fixtures function scoped and add new tests --- .../ide_event_handler/file_manager.py | 2 +- .../ide_event_handler/test_filemanager.py | 138 +++++++++--------- 2 files changed, 73 insertions(+), 67 deletions(-) diff --git a/lib/tfw/components/ide_event_handler/file_manager.py b/lib/tfw/components/ide_event_handler/file_manager.py index 59431ea..967e06f 100644 --- a/lib/tfw/components/ide_event_handler/file_manager.py +++ b/lib/tfw/components/ide_event_handler/file_manager.py @@ -41,7 +41,7 @@ class FileManager: # pylint: disable=too-many-instance-attributes @allowed_directories.setter def allowed_directories(self, directories): - self._allowed_directories = directories + self._allowed_directories = [realpath(directory) for directory in directories] @property def filename(self): diff --git a/lib/tfw/components/ide_event_handler/test_filemanager.py b/lib/tfw/components/ide_event_handler/test_filemanager.py index d62f3e0..c6e0b8b 100644 --- a/lib/tfw/components/ide_event_handler/test_filemanager.py +++ b/lib/tfw/components/ide_event_handler/test_filemanager.py @@ -1,97 +1,103 @@ # pylint: disable=redefined-outer-name from secrets import token_urlsafe +from os.path import join +from os import chdir, mkdir, symlink from pathlib import Path -from shutil import rmtree -from os.path import join, realpath -from os import mkdir, rmdir, remove, symlink +from tempfile import TemporaryDirectory import pytest -from filemanager import FileManager +from file_manager import FileManager -WORKDIR = realpath('test_filemanager') +class ManagerContext: + def __init__(self, folder, manager): + self.folder = folder + self.manager = manager -def workdir_pref(path): - return join(WORKDIR, path) + def join(self, path): + return join(self.folder, path) -@pytest.fixture(scope='module') -def manager(): + +@pytest.fixture() +def context(): dirs = [] - mkdir(WORKDIR) - for i in range(3): - node = workdir_pref('dir_'+str(i).zfill(2)) - mkdir(node) - Path(join(node, 'empty.txt')).touch() - Path(join(node, 'empty.bin')).touch() - dirs.append(node) + with TemporaryDirectory() as workdir: + chdir(workdir) + for name in ["allowed", "excluded", "invis"]: + node = join(workdir, name) + mkdir(node) + Path(join(node, 'empty.txt')).touch() + Path(join(node, 'empty.bin')).touch() + dirs.append(node) - yield FileManager(dirs[0], dirs[:-1], exclude=['*/dir_01/*', '*.bin']) - rmtree(WORKDIR) + yield ManagerContext( + workdir, + FileManager(dirs[0], dirs[:-1], exclude=['*/excluded/*']) + ) -@pytest.mark.parametrize('subdir', ['dir_00', 'dir_01']) -def test_select_allowed_dirs(manager, subdir): - manager.workdir = workdir_pref(subdir) - assert manager.workdir == workdir_pref(subdir) - newdir = workdir_pref(join(subdir, 'deep')) +@pytest.mark.parametrize('subdir', ['allowed/', 'excluded/']) +def test_select_allowed_dirs(context, subdir): + context.manager.workdir = context.join(subdir) + assert context.manager.workdir == context.join(subdir) + newdir = context.join(subdir+'deep') mkdir(newdir) - manager.workdir = newdir - assert manager.workdir == newdir - rmdir(newdir) + context.manager.workdir = newdir + assert context.manager.workdir == newdir -@pytest.mark.parametrize('excdir', ['/', workdir_pref('dir_02')]) -def test_select_excluded_dirs(manager, excdir): - allowed = manager.allowed_directories +@pytest.mark.parametrize('invdir', ['', 'invis']) +def test_select_forbidden_dirs(context, invdir): + fullpath = context.join(invdir) with pytest.raises(OSError): - manager.workdir = excdir - assert manager.workdir != excdir - manager.allowed_directories = allowed+[excdir] - manager.workdir = excdir - assert manager.workdir == excdir - manager.allowed_directories = allowed + context.manager.workdir = fullpath + assert context.manager.workdir != fullpath + context.manager.allowed_directories += [fullpath] + context.manager.workdir = fullpath + assert context.manager.workdir == fullpath + @pytest.mark.parametrize('filename', ['another.txt', '*.txt']) -def test_select_allowed_files(manager, filename): - manager.workdir = workdir_pref('dir_00') - newfile = workdir_pref(join('dir_00', filename)) - Path(newfile).touch() - assert filename in manager.files - manager.filename = filename - assert manager.filename == filename - remove(newfile) +def test_select_allowed_files(context, filename): + Path(context.join('allowed/'+filename)).touch() + assert filename in context.manager.files + context.manager.filename = filename + assert context.manager.filename == filename @pytest.mark.parametrize('path', [ - ['dir_00', 'illegal.bin'], - ['dir_01', 'legal.txt'] + {'dir': 'allowed/', 'file': 'illegal.bin'}, + {'dir': 'excluded/', 'file': 'legal.txt'}, + {'dir': 'allowed/', 'file': token_urlsafe(16)+'.bin'}, + {'dir': 'excluded/', 'file': token_urlsafe(16)+'.txt'}, + {'dir': 'allowed/', 'file': token_urlsafe(32)+'.bin'}, + {'dir': 'excluded/', 'file': token_urlsafe(32)+'.txt'} ]) -def test_select_excluded_files(manager, path): - manager.workdir = workdir_pref(path[0]) - newfile = workdir_pref(join(path[0], path[1])) - Path(newfile).touch() - assert path[1] not in manager.files +def test_select_excluded_files(context, path): + context.manager.workdir = context.join(path['dir']) + context.manager.exclude = ['*/excluded/*', '*.bin'] + Path(context.join(path['dir']+path['file'])).touch() + assert path['file'] not in context.manager.files with pytest.raises(OSError): - manager.filename = path[1] - remove(newfile) + context.manager.filename = path['file'] @pytest.mark.parametrize('path', [ - ['dir_02/empty.txt', 'dir_00/link.txt'], - ['dir_01/empty.txt', 'dir_00/link.bin'] + {'src': 'excluded/empty.txt', 'dst': 'allowed/link.txt'}, + {'src': 'invis/empty.txt', 'dst': 'allowed/link.txt'}, + {'src': 'excluded/empty.txt', 'dst': 'allowed/'+token_urlsafe(16)+'.txt'}, + {'src': 'invis/empty.txt', 'dst': 'allowed/'+token_urlsafe(16)+'.txt'}, + {'src': 'excluded/empty.txt', 'dst': 'allowed/'+token_urlsafe(32)+'.txt'}, + {'src': 'invis/empty.txt', 'dst': 'allowed/'+token_urlsafe(32)+'.txt'} ]) -def test_select_excluded_symlinks(manager, path): - manager.workdir = workdir_pref('dir_00') - link = workdir_pref(path[1]) - symlink(workdir_pref(path[0]), link) - assert path[1] not in manager.files - remove(link) +def test_select_excluded_symlinks(context, path): + symlink(context.join(path['src']), context.join(path['dst'])) + assert path['dst'] not in context.manager.files -def test_read_write_file(manager): +def test_read_write_file(context): for _ in range(128): - manager.workdir = workdir_pref('dir_00') - manager.filename = 'empty.txt' + context.manager.filename = 'empty.txt' content = token_urlsafe(32) - manager.file_contents = content - assert manager.file_contents == content - with open(workdir_pref('dir_00/empty.txt'), "r") as ifile: + context.manager.file_contents = content + assert context.manager.file_contents == content + with open(context.join('allowed/empty.txt'), "r") as ifile: assert ifile.read() == content From 8cbe737d2fc46c21321e8f257cd60727f26cccad Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 6 Jun 2019 09:58:18 +0200 Subject: [PATCH 025/174] Turn context into a dataclass and add new test case --- ...st_filemanager.py => test_file_manager.py} | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) rename lib/tfw/components/ide_event_handler/{test_filemanager.py => test_file_manager.py} (74%) diff --git a/lib/tfw/components/ide_event_handler/test_filemanager.py b/lib/tfw/components/ide_event_handler/test_file_manager.py similarity index 74% rename from lib/tfw/components/ide_event_handler/test_filemanager.py rename to lib/tfw/components/ide_event_handler/test_file_manager.py index c6e0b8b..33c8024 100644 --- a/lib/tfw/components/ide_event_handler/test_filemanager.py +++ b/lib/tfw/components/ide_event_handler/test_file_manager.py @@ -1,5 +1,6 @@ # pylint: disable=redefined-outer-name +from dataclasses import dataclass from secrets import token_urlsafe from os.path import join from os import chdir, mkdir, symlink @@ -10,11 +11,10 @@ import pytest from file_manager import FileManager - +@dataclass class ManagerContext: - def __init__(self, folder, manager): - self.folder = folder - self.manager = manager + folder: str + manager: FileManager def join(self, path): return join(self.folder, path) @@ -22,20 +22,24 @@ class ManagerContext: @pytest.fixture() def context(): - dirs = [] + dirs = {} with TemporaryDirectory() as workdir: chdir(workdir) - for name in ["allowed", "excluded", "invis"]: + for name in ['allowed', 'excluded', 'invis']: node = join(workdir, name) mkdir(node) Path(join(node, 'empty.txt')).touch() Path(join(node, 'empty.bin')).touch() - dirs.append(node) + dirs[name] = node yield ManagerContext( workdir, - FileManager(dirs[0], dirs[:-1], exclude=['*/excluded/*']) + FileManager( + dirs['allowed'], + [dirs['allowed'], dirs['excluded']], + exclude=['*/excluded/*'] + ) ) @pytest.mark.parametrize('subdir', ['allowed/', 'excluded/']) @@ -99,5 +103,22 @@ def test_read_write_file(context): content = token_urlsafe(32) context.manager.file_contents = content assert context.manager.file_contents == content - with open(context.join('allowed/empty.txt'), "r") as ifile: + with open(context.join('allowed/empty.txt'), 'r') as ifile: assert ifile.read() == content + +def test_regular_ide_actions(context): + context.manager.workdir = context.join('allowed') + newfile1, newfile2 = token_urlsafe(16), token_urlsafe(16) + Path(context.join(f'allowed/{newfile1}')).touch() + Path(context.join(f'allowed/{newfile2}')).touch() + for _ in range(8): + context.manager.filename = newfile1 + content1 = token_urlsafe(32) + context.manager.file_contents = content1 + context.manager.filename = newfile2 + content2 = token_urlsafe(32) + context.manager.file_contents = content2 + context.manager.filename = newfile1 + assert context.manager.file_contents == content1 + context.manager.filename = newfile2 + assert context.manager.file_contents == content2 From d2131682a8cafd185391aa40d7b8eb49dba86570 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 11 Jun 2019 11:21:46 +0200 Subject: [PATCH 026/174] Make recursive watching optional and rename reset() --- lib/tfw/components/inotify/inotify.py | 14 +++++++------- lib/tfw/components/inotify/test_inotify.py | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/tfw/components/inotify/inotify.py b/lib/tfw/components/inotify/inotify.py index f26ce6b..2cf9523 100644 --- a/lib/tfw/components/inotify/inotify.py +++ b/lib/tfw/components/inotify/inotify.py @@ -64,21 +64,21 @@ class InotifyDirDeletedEvent(InotifyEvent): class InotifyObserver: - def __init__(self, path, patterns=None, exclude=None): + def __init__(self, path, patterns=None, exclude=None, recursive=False): self._path = path self._patterns = patterns self._exclude = exclude self.observer = Observer() - self.reset(path, patterns, exclude) + self._reset(path, patterns, exclude) - def reset(self, path, patterns, exclude): + def _reset(self, path, patterns, exclude): dispatch_event = self.dispatch_event class TransformerEventHandler(PatternMatchingEventHandler): def on_any_event(self, event): dispatch_event(event) self.handler = TransformerEventHandler(patterns, exclude) self.observer.unschedule_all() - self.observer.schedule(self.handler, path, recursive=True) + self.observer.schedule(self.handler, path, recursive) @property def path(self): @@ -87,7 +87,7 @@ class InotifyObserver: @path.setter def path(self, path): self._path = path - self.reset(path, self._patterns, self._exclude) + self._reset(path, self._patterns, self._exclude) @property def patterns(self): @@ -96,7 +96,7 @@ class InotifyObserver: @patterns.setter def patterns(self, patterns): self._patterns = patterns - self.reset(self._path, patterns, self._exclude) + self._reset(self._path, patterns, self._exclude) @property def exclude(self): @@ -105,7 +105,7 @@ class InotifyObserver: @exclude.setter def exclude(self, exclude): self._exclude = exclude - self.reset(self._path, self._patterns, exclude) + self._reset(self._path, self._patterns, exclude) def start(self): self.observer.start() diff --git a/lib/tfw/components/inotify/test_inotify.py b/lib/tfw/components/inotify/test_inotify.py index 702e2c7..6b4bdb3 100644 --- a/lib/tfw/components/inotify/test_inotify.py +++ b/lib/tfw/components/inotify/test_inotify.py @@ -66,10 +66,10 @@ class InotifyContext: class InotifyTestObserver(InotifyObserver): - def __init__(self, path, patterns=None, exclude=None): + def __init__(self, path, patterns=None, exclude=None, recursive=False): self.any_list = [] self.create_queue, self.modify_queue, self.move_queue, self.delete_queue = [Queue() for _ in range(4)] - super().__init__(path, patterns, exclude) + super().__init__(path, patterns, exclude, recursive) def on_any_event(self, event): self.any_list.append(event) @@ -99,7 +99,7 @@ def context(): mkdir(newdir) Path(join(newdir, generate_name()+'.txt')).touch() - monitor = InotifyTestObserver(workdir) + monitor = InotifyTestObserver(workdir, recursive=True) monitor.start() yield InotifyContext(workdir, monitor) From 20bfe3c352f54e53e22f8bdbb137358bd01ee66b Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 11 Jun 2019 13:01:56 +0200 Subject: [PATCH 027/174] Make recursive watching optional and underscore private attributes/methods --- lib/tfw/components/inotify/inotify.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/tfw/components/inotify/inotify.py b/lib/tfw/components/inotify/inotify.py index 2cf9523..33b0936 100644 --- a/lib/tfw/components/inotify/inotify.py +++ b/lib/tfw/components/inotify/inotify.py @@ -68,7 +68,8 @@ class InotifyObserver: self._path = path self._patterns = patterns self._exclude = exclude - self.observer = Observer() + self._recursive = recursive + self._observer = Observer() self._reset(path, patterns, exclude) def _reset(self, path, patterns, exclude): @@ -77,8 +78,8 @@ class InotifyObserver: def on_any_event(self, event): dispatch_event(event) self.handler = TransformerEventHandler(patterns, exclude) - self.observer.unschedule_all() - self.observer.schedule(self.handler, path, recursive) + self._observer.unschedule_all() + self._observer.schedule(self.handler, path, self._recursive) @property def path(self): @@ -108,11 +109,11 @@ class InotifyObserver: self._reset(self._path, self._patterns, exclude) def start(self): - self.observer.start() + self._observer.start() def stop(self): - self.observer.stop() - self.observer.join() + self._observer.stop() + self._observer.join() def dispatch_event(self, event): event_to_action = { From 3cc70c2147a50e84704b3ee6a23f2b66d594719e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 17 Jun 2019 15:09:42 +0200 Subject: [PATCH 028/174] Fix IdeEventHandler broken import --- lib/tfw/components/ide_event_handler/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 lib/tfw/components/ide_event_handler/__init__.py diff --git a/lib/tfw/components/ide_event_handler/__init__.py b/lib/tfw/components/ide_event_handler/__init__.py new file mode 100644 index 0000000..d597b12 --- /dev/null +++ b/lib/tfw/components/ide_event_handler/__init__.py @@ -0,0 +1 @@ +from .ide_event_handler import IdeEventHandler From 49856cebe2b7e5fb68289a83dfbec5b9f1b50d2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 18 Jun 2019 14:54:54 +0200 Subject: [PATCH 029/174] Improve InotifyEvent.__repr__ --- lib/tfw/components/inotify/inotify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tfw/components/inotify/inotify.py b/lib/tfw/components/inotify/inotify.py index 33b0936..032fd67 100644 --- a/lib/tfw/components/inotify/inotify.py +++ b/lib/tfw/components/inotify/inotify.py @@ -19,7 +19,7 @@ class InotifyEvent: return self.__repr__() def __repr__(self): - return f'{self.date}\t{self.__class__.__name__}:\t{self.src_path}' + return f'{self.__class__.__name__}({self.src_path})' class InotifyMovedEvent(InotifyEvent): @@ -28,7 +28,7 @@ class InotifyMovedEvent(InotifyEvent): super().__init__(src_path) def __repr__(self): - return super().__repr__()+f'\t-> {self.dest_path}' + return f'{self.__class__.__name__}({self.src_path}, {self.dest_path})' class InotifyFileCreatedEvent(InotifyEvent): From 85c720127e57a9cc7b4e026f741b7a5c1cbddad4 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 11 Jun 2019 17:25:35 +0200 Subject: [PATCH 030/174] Make HistoryMonitor utilize our inotify module --- lib/tfw/components/history_monitor.py | 53 +++++++++----------- lib/tfw/components/terminal_commands.py | 4 +- lib/tfw/components/terminal_event_handler.py | 2 +- 3 files changed, 27 insertions(+), 32 deletions(-) diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index 410558a..a11583a 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -6,25 +6,11 @@ from re import findall from re import compile as compileregex from abc import ABC, abstractmethod -from watchdog.events import PatternMatchingEventHandler - -from tfw.mixins.callback_mixin import CallbackMixin -from tfw.mixins.observer_mixin import ObserverMixin -from tfw.decorators.rate_limiter import RateLimiter +from tfw.components.inotify import InotifyObserver +from tfw.event_handlers import TFWServerUplinkConnector -class CallbackEventHandler(PatternMatchingEventHandler, ABC): - def __init__(self, files, *callbacks): - super().__init__(files) - self.callbacks = callbacks - - @RateLimiter(rate_per_second=2) - def on_modified(self, event): - for callback in self.callbacks: - callback() - - -class HistoryMonitor(CallbackMixin, ObserverMixin, ABC): +class HistoryMonitor(ABC, InotifyObserver): """ Abstract class capable of monitoring and parsing a history file such as bash HISTFILEs. Monitoring means detecting when the file was changed and @@ -36,18 +22,19 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC): command pattern property and optionally the sanitize_command method. See examples below. """ - def __init__(self, histfile): + def __init__(self, domain, histfile): + self.domain = domain self.histfile = histfile self._history = [] self._last_length = len(self._history) - self.observer.schedule( - CallbackEventHandler( - [self.histfile], - self._fetch_history, - self._invoke_callbacks - ), - dirname(self.histfile) - ) + self.uplink = TFWServerUplinkConnector() + super().__init__(dirname(self.histfile), [self.histfile]) + + def on_modified(self, event): + self._fetch_history() + if self._last_length < len(self._history): + for command in self._history[self._last_length:]: + self.send_message(command) @property def history(self): @@ -72,9 +59,11 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC): # pylint: disable=no-self-use return command - def _invoke_callbacks(self): - if self._last_length < len(self._history): - self._execute_callbacks(self.history) + def send_message(self, command): + self.uplink.send_message({ + 'key': f'history.{self.domain}', + 'value': command + }) class BashMonitor(HistoryMonitor): @@ -87,6 +76,9 @@ class BashMonitor(HistoryMonitor): shopt -s histappend unset HISTCONTROL """ + def __init__(self, histfile): + super().__init__('bash', histfile) + @property def command_pattern(self): return r'.+' @@ -100,6 +92,9 @@ class GDBMonitor(HistoryMonitor): HistoryMonitor to monitor GDB sessions. For this to work "set trace-commands on" must be set in GDB. """ + def __init__(self, histfile): + super().__init__('gdb', histfile) + @property def command_pattern(self): return r'(?<=\n)\+(.+)\n' diff --git a/lib/tfw/components/terminal_commands.py b/lib/tfw/components/terminal_commands.py index 9a55d22..e536c92 100644 --- a/lib/tfw/components/terminal_commands.py +++ b/lib/tfw/components/terminal_commands.py @@ -61,8 +61,8 @@ class TerminalCommands(ABC): def _match_command_regex(self, string): return match(self._command_method_regex, string) - def callback(self, history): - parts = split(history[-1]) + def callback(self, command): + parts = split(command) command = parts[0] if command in self.command_implemetations.keys(): try: diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/components/terminal_event_handler.py index 436247f..c7104e0 100644 --- a/lib/tfw/components/terminal_event_handler.py +++ b/lib/tfw/components/terminal_event_handler.py @@ -43,7 +43,7 @@ class TerminalEventHandler(FrontendEventHandlerBase): } if self._historymonitor: - self._historymonitor.watch() + self._historymonitor.start() self.terminado_server.listen() @property From 8bf18113b2c247f27e9fb1e304ace97fcc7d8e8d Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 18 Jun 2019 18:43:02 +0200 Subject: [PATCH 031/174] Implement custom logging logic --- lib/tfw/config/log.py | 116 ++++++++++++++++++++++++++ supervisor/components/tfw_server.conf | 2 +- supervisor/tfw_server.py | 6 +- 3 files changed, 119 insertions(+), 5 deletions(-) create mode 100644 lib/tfw/config/log.py diff --git a/lib/tfw/config/log.py b/lib/tfw/config/log.py new file mode 100644 index 0000000..332fef0 --- /dev/null +++ b/lib/tfw/config/log.py @@ -0,0 +1,116 @@ +# pylint: disable=bad-whitespace +from sys import stderr +from datetime import datetime +from logging import DEBUG, getLogger, Handler, Formatter, Filter + +TFW_LOG_PATH = '/var/log/tfw.log' + + +class COLOR: + BLACK = '\033[30m' + GREY = '\033[30;1m' + RED = '\033[31m' + BOLDRED = '\033[31;1m' + GREEN = '\033[32m' + BOLDGREEN = '\033[32;1m' + ORANGE = '\033[33m' + YELLOW = '\033[33;1m' + BLUE = '\033[34m' + BOLDBLUE = '\033[34;1m' + MAGENTA = '\033[35m' + BOLDMAGENTA = '\033[35;1m' + CYAN = '\033[36m' + BOLDCYAN = '\033[36;1m' + WHITE = '\033[37m' + BOLDWHITE = '\033[37;1m' + RESET = '\033[0m' + + +class TFWLog: + def __init__(self, path=TFW_LOG_PATH, level=DEBUG): + self.log = getLogger() + self.old_level = self.log.level + self.new_level = level + self.handler = TFWLogHandler(path) + self.handler.setFormatter(TFWLogFormatter()) + + def start(self): + self.log.setLevel(self.new_level) + self.log.addHandler(self.handler) + self.log.info('Logging started.') + + def stop(self): + self.log.info('Stop logging.') + self.log.setLevel(self.old_level) + self.handler.close() + self.log.removeHandler(self.handler) + + +class TFWLogHandler(Handler): + def __init__(self, path): + self.logfile = open(path, 'a+') + super().__init__() + + def emit(self, record): + short_entry, long_entry = self.format(record) + stderr.write(short_entry+'\n') + self.logfile.write(long_entry+'\n') + stderr.flush() + self.logfile.flush() + + def close(self): + self.logfile.close() + +class TFWLogFormatter(Formatter): + severity = { + 'CRITICAL' : COLOR.BOLDRED, + 'ERROR' : COLOR.RED, + 'WARNING' : COLOR.YELLOW, + 'INFO' : COLOR.BOLDGREEN, + 'DEBUG' : COLOR.BOLDWHITE, + 'NOTSET' : COLOR.CYAN + } + + def format(self, record): + date = datetime.utcfromtimestamp(record.created).strftime('%H:%M:%S') + if record.args: + tuple_args = (record.args,) if isinstance(record.args, dict) else record.args + clean_args = tuple((self.trim(arg) for arg in tuple_args)) + short_message = record.msg % clean_args + long_message = record.msg % record.args + else: + short_message = record.msg + long_message = record.msg + + short_entry = (f'[{COLOR.GREY}{date}{COLOR.RESET}|>' + f'{self.severity[record.levelname]}{record.module}:' + f'{record.levelname.lower()}{COLOR.RESET}] {short_message}') + long_entry = f'[{date}|>{record.module}:{record.levelname.lower()}] {long_message}' + return short_entry, long_entry + + def trim(self, value, in_dict=False): + if isinstance(value, dict): + trimmed = {k: self.trim(v, True) for k, v in value.items()} + return trimmed if in_dict else str(trimmed) + if isinstance(value, (int, float)): + return value + value_str = str(value) + return value_str if len(value_str) <= 20 else f'{value_str[:20]}...' + + +class TFWLogWhitelistFilter(Filter): + def __init__(self, names): + self.names = names + super().__init__() + + def filter(self, record): + return record.module in self.names + + +class TFWLogBlacklistFilter(Filter): + def __init__(self, names): + self.names = names + super().__init__() + + def filter(self, record): + return record.module not in self.names diff --git a/supervisor/components/tfw_server.conf b/supervisor/components/tfw_server.conf index f583be9..a131674 100644 --- a/supervisor/components/tfw_server.conf +++ b/supervisor/components/tfw_server.conf @@ -1,4 +1,4 @@ [program:tfwserver] user=root directory=%(ENV_TFW_SERVER_DIR)s -command=python3 tfw_server.py +command=python3 -u tfw_server.py diff --git a/supervisor/tfw_server.py b/supervisor/tfw_server.py index 0d52bbf..35fc2fa 100644 --- a/supervisor/tfw_server.py +++ b/supervisor/tfw_server.py @@ -1,12 +1,10 @@ -import logging - from tornado.ioloop import IOLoop +from tfw.config.log import TFWLog from tfw.server import TFWServer -logging.basicConfig(level=logging.DEBUG) - if __name__ == '__main__': + TFWLog().start() TFWServer().listen() IOLoop.instance().start() From c6e0b5493067a103975de6a3171113560cff7bb8 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 19 Jun 2019 15:26:04 +0200 Subject: [PATCH 032/174] Make Inotify testing faster --- lib/tfw/components/inotify/test_inotify.py | 145 ++++++++++----------- 1 file changed, 66 insertions(+), 79 deletions(-) diff --git a/lib/tfw/components/inotify/test_inotify.py b/lib/tfw/components/inotify/test_inotify.py index 6b4bdb3..b6e659e 100644 --- a/lib/tfw/components/inotify/test_inotify.py +++ b/lib/tfw/components/inotify/test_inotify.py @@ -9,6 +9,7 @@ from os import chdir, mkdir, rename, walk, listdir from tempfile import TemporaryDirectory import pytest +import watchdog from inotify import InotifyObserver from inotify import ( @@ -17,11 +18,15 @@ from inotify import ( InotifyDirMovedEvent, InotifyDirDeletedEvent ) +watchdog.observers.inotify_buffer.InotifyBuffer.delay = 0 + class InotifyContext: - def __init__(self, folder, observer): + def __init__(self, workdir, subdir, subfile, observer): self.missing_events = 0 - self.folder = folder + self.workdir = workdir + self.subdir = subdir + self.subfile = subfile self.observer = observer self.event_to_queue = { @@ -46,18 +51,18 @@ class InotifyContext: return dirname def join(self, path): - return join(self.folder, path) + return join(self.workdir, path) def check_event(self, event_type, path): self.missing_events += 1 - event = self.event_to_queue[event_type].get(timeout=1) + event = self.event_to_queue[event_type].get(timeout=0.1) assert isinstance(event, event_type) assert event.src_path == path return event def check_empty(self, event_type): with pytest.raises(Empty): - self.event_to_queue[event_type].get(timeout=1) + self.event_to_queue[event_type].get(timeout=0.1) def check_any(self): attrs = self.observer.__dict__.values() @@ -92,95 +97,77 @@ def generate_name(): @pytest.fixture() def context(): with TemporaryDirectory() as workdir: - chdir(workdir) - - for _ in range(5): - newdir = join(workdir, generate_name()) - mkdir(newdir) - Path(join(newdir, generate_name()+'.txt')).touch() - + subdir = join(workdir, generate_name()) + subfile = join(subdir, generate_name()+'.txt') + mkdir(subdir) + Path(subfile).touch() monitor = InotifyTestObserver(workdir, recursive=True) monitor.start() - yield InotifyContext(workdir, monitor) + yield InotifyContext(workdir, subdir, subfile, monitor) def test_create(context): - for _, dirs, _ in list(walk(context.folder)): - for name in dirs: - newfile = context.create_random_file(name, '.txt') - context.check_event(InotifyFileCreatedEvent, newfile) - newdir = context.create_random_folder(name) - context.check_event(InotifyDirCreatedEvent, newdir) - assert context.check_any() + newfile = context.create_random_file(context.workdir, '.txt') + context.check_event(InotifyFileCreatedEvent, newfile) + newdir = context.create_random_folder(context.workdir) + context.check_event(InotifyDirCreatedEvent, newdir) + assert context.check_any() def test_modify(context): - for root, _, files in list(walk(context.folder)): - for name in files: - oldfile = join(root, name) - with open(oldfile, 'w') as ofile: - ofile.write('text') - context.check_event(InotifyFileModifiedEvent, oldfile) - rename(oldfile, oldfile+'_new') - context.check_event(InotifyDirModifiedEvent, root) - assert context.check_any() + with open(context.subfile, 'w') as ofile: + ofile.write('text') + context.check_event(InotifyFileModifiedEvent, context.subfile) + while True: + try: + context.observer.modify_queue.get(timeout=0.1) + context.missing_events += 1 + except Empty: + break + rename(context.subfile, context.subfile+'_new') + context.check_event(InotifyDirModifiedEvent, context.subdir) + assert context.check_any() def test_move(context): - for root, dirs, _ in list(walk(context.folder)): - for name in dirs: - olddir = join(root, name) - oldfile = join(olddir, list(listdir(olddir))[0]) - rename(olddir, olddir+'_new') - context.check_event(InotifyDirMovedEvent, olddir) - context.check_event(InotifyFileMovedEvent, oldfile) - assert context.check_any() + rename(context.subdir, context.subdir+'_new') + context.check_event(InotifyDirMovedEvent, context.subdir) + context.check_event(InotifyFileMovedEvent, context.subfile) + assert context.check_any() def test_delete(context): - for root, dirs, _ in list(walk(context.folder)): - for name in dirs: - olddir = join(root, name) - oldfile = join(olddir, list(listdir(olddir))[0]) - rmtree(olddir) - context.check_event(InotifyFileDeletedEvent, oldfile) - context.check_event(InotifyDirDeletedEvent, olddir) - assert context.check_any() + rmtree(context.subdir) + context.check_event(InotifyFileDeletedEvent, context.subfile) + context.check_event(InotifyDirDeletedEvent, context.subdir) + assert context.check_any() def test_path(context): - for _, dirs, _ in list(walk(context.folder)): - for name in dirs: - context.observer.path = context.join(name) - context.create_random_folder('.') - newfile = context.create_random_file(name, '.txt') - context.check_event(InotifyFileCreatedEvent, newfile) - context.observer.path = context.folder - assert context.check_any() + context.observer.path = context.subdir + context.create_random_folder(context.workdir) + newfile = context.create_random_file(context.subdir, '.txt') + context.check_event(InotifyFileCreatedEvent, newfile) + context.observer.path = context.subdir + assert context.check_any() def test_patterns(context): - for _, dirs, _ in list(walk(context.folder)): - for name in dirs: - context.observer.patterns = ["*.txt"] - context.create_random_file(name, '.bin') - newfile = context.create_random_file(name, '.txt') - context.check_event(InotifyFileCreatedEvent, newfile) - context.check_empty(InotifyFileCreatedEvent) - assert context.check_any() - context.observer.patterns = None + context.observer.patterns = ['*.txt'] + context.create_random_file(context.subdir, '.bin') + newfile = context.create_random_file(context.subdir, '.txt') + context.check_event(InotifyFileCreatedEvent, newfile) + context.check_empty(InotifyFileCreatedEvent) + assert context.check_any() + context.observer.patterns = None def test_exclude(context): - for _, dirs, _ in list(walk(context.folder)): - for name in dirs: - context.observer.exclude = ["*.txt"] - context.create_random_file(name, '.txt') - newfile = context.create_random_file(name, '.bin') - context.check_event(InotifyFileCreatedEvent, newfile) - context.check_empty(InotifyFileCreatedEvent) - assert context.check_any() - context.observer.exclude = None + context.observer.exclude = ['*.txt'] + context.create_random_file(context.subdir, '.txt') + newfile = context.create_random_file(context.subdir, '.bin') + context.check_event(InotifyFileCreatedEvent, newfile) + context.check_empty(InotifyFileCreatedEvent) + assert context.check_any() + context.observer.exclude = None def test_stress(context): - for _, dirs, _ in list(walk(context.folder)): - for name in dirs: - newfile = [] - for i in range(1024): - newfile.append(context.create_random_file(name, '.txt')) - for i in range(1024): - context.check_event(InotifyFileCreatedEvent, newfile[i]) - assert context.check_any() + newfile = [] + for i in range(1024): + newfile.append(context.create_random_file(context.subdir, '.txt')) + for i in range(1024): + context.check_event(InotifyFileCreatedEvent, newfile[i]) + assert context.check_any() From c8a0546041a1aed9de5a38c668fb4b97a2517914 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 19 Jun 2019 16:13:01 +0200 Subject: [PATCH 033/174] Remove unnecessary imports --- lib/tfw/components/inotify/test_inotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/components/inotify/test_inotify.py b/lib/tfw/components/inotify/test_inotify.py index b6e659e..e1e72c8 100644 --- a/lib/tfw/components/inotify/test_inotify.py +++ b/lib/tfw/components/inotify/test_inotify.py @@ -5,7 +5,7 @@ from secrets import token_urlsafe from pathlib import Path from shutil import rmtree from os.path import join -from os import chdir, mkdir, rename, walk, listdir +from os import mkdir, rename from tempfile import TemporaryDirectory import pytest From 13e7470f480a8988b0c166d9c5af90a51eb647ec Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 20 Jun 2019 16:04:21 +0200 Subject: [PATCH 034/174] Separate event handlers from independent components --- lib/tfw/builtins/__init__.py | 10 ++++++++++ .../directory_monitoring_event_handler.py | 0 .../directory_snapshotting_event_handler.py | 0 .../frontend_event_handler.py | 3 +-- .../fsm_managing_event_handler.py | 0 .../ide_event_handler.py | 3 +-- .../log_monitoring_event_handler.py | 0 .../pipe_io_event_handler.py | 3 +-- .../process_managing_event_handler.py | 0 .../terminal_event_handler.py | 0 lib/tfw/components/__init__.py | 17 ++++------------- lib/tfw/components/file_manager/__init__.py | 1 + .../file_manager.py | 0 .../test_file_manager.py | 0 .../components/ide_event_handler/__init__.py | 1 - 15 files changed, 18 insertions(+), 20 deletions(-) create mode 100644 lib/tfw/builtins/__init__.py rename lib/tfw/{components => builtins}/directory_monitoring_event_handler.py (100%) rename lib/tfw/{components => builtins}/directory_snapshotting_event_handler.py (100%) rename lib/tfw/{components => builtins}/frontend_event_handler.py (96%) rename lib/tfw/{components => builtins}/fsm_managing_event_handler.py (100%) rename lib/tfw/{components/ide_event_handler => builtins}/ide_event_handler.py (99%) rename lib/tfw/{components => builtins}/log_monitoring_event_handler.py (100%) rename lib/tfw/{components => builtins}/pipe_io_event_handler.py (98%) rename lib/tfw/{components => builtins}/process_managing_event_handler.py (100%) rename lib/tfw/{components => builtins}/terminal_event_handler.py (100%) create mode 100644 lib/tfw/components/file_manager/__init__.py rename lib/tfw/components/{ide_event_handler => file_manager}/file_manager.py (100%) rename lib/tfw/components/{ide_event_handler => file_manager}/test_file_manager.py (100%) delete mode 100644 lib/tfw/components/ide_event_handler/__init__.py diff --git a/lib/tfw/builtins/__init__.py b/lib/tfw/builtins/__init__.py new file mode 100644 index 0000000..7e94f10 --- /dev/null +++ b/lib/tfw/builtins/__init__.py @@ -0,0 +1,10 @@ +from .directory_monitoring_event_handler import DirectoryMonitoringEventHandler +from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler +from .frontend_event_handler import FrontendEventHandler +from .fsm_managing_event_handler import FSMManagingEventHandler +from .ide_event_handler import IdeEventHandler +from .log_monitoring_event_handler import LogMonitoringEventHandler +from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler, PipeIOServer +from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler +from .process_managing_event_handler import ProcessManagingEventHandler +from .terminal_event_handler import TerminalEventHandler diff --git a/lib/tfw/components/directory_monitoring_event_handler.py b/lib/tfw/builtins/directory_monitoring_event_handler.py similarity index 100% rename from lib/tfw/components/directory_monitoring_event_handler.py rename to lib/tfw/builtins/directory_monitoring_event_handler.py diff --git a/lib/tfw/components/directory_snapshotting_event_handler.py b/lib/tfw/builtins/directory_snapshotting_event_handler.py similarity index 100% rename from lib/tfw/components/directory_snapshotting_event_handler.py rename to lib/tfw/builtins/directory_snapshotting_event_handler.py diff --git a/lib/tfw/components/frontend_event_handler.py b/lib/tfw/builtins/frontend_event_handler.py similarity index 96% rename from lib/tfw/components/frontend_event_handler.py rename to lib/tfw/builtins/frontend_event_handler.py index 1a76abe..b372023 100644 --- a/lib/tfw/components/frontend_event_handler.py +++ b/lib/tfw/builtins/frontend_event_handler.py @@ -1,10 +1,9 @@ from abc import ABC, abstractmethod from contextlib import suppress +from tfw.components.message_sender import MessageSender from tfw.event_handlers import FrontendEventHandlerBase -from .message_sender import MessageSender - class FrontendEventHandler(FrontendEventHandlerBase): def __init__(self): diff --git a/lib/tfw/components/fsm_managing_event_handler.py b/lib/tfw/builtins/fsm_managing_event_handler.py similarity index 100% rename from lib/tfw/components/fsm_managing_event_handler.py rename to lib/tfw/builtins/fsm_managing_event_handler.py diff --git a/lib/tfw/components/ide_event_handler/ide_event_handler.py b/lib/tfw/builtins/ide_event_handler.py similarity index 99% rename from lib/tfw/components/ide_event_handler/ide_event_handler.py rename to lib/tfw/builtins/ide_event_handler.py index 218d3ed..e91213b 100644 --- a/lib/tfw/components/ide_event_handler/ide_event_handler.py +++ b/lib/tfw/builtins/ide_event_handler.py @@ -9,10 +9,9 @@ from typing import Iterable from tfw.event_handlers import FrontendEventHandlerBase from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin +from tfw.components import FileManager from tfw.components.directory_monitor import DirectoryMonitor -from .file_manager import FileManager - LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/log_monitoring_event_handler.py b/lib/tfw/builtins/log_monitoring_event_handler.py similarity index 100% rename from lib/tfw/components/log_monitoring_event_handler.py rename to lib/tfw/builtins/log_monitoring_event_handler.py diff --git a/lib/tfw/components/pipe_io_event_handler.py b/lib/tfw/builtins/pipe_io_event_handler.py similarity index 98% rename from lib/tfw/components/pipe_io_event_handler.py rename to lib/tfw/builtins/pipe_io_event_handler.py index da3e726..9ab4db5 100644 --- a/lib/tfw/components/pipe_io_event_handler.py +++ b/lib/tfw/builtins/pipe_io_event_handler.py @@ -11,8 +11,7 @@ from threading import Thread from contextlib import suppress from tfw.event_handlers import EventHandlerBase - -from .pipe_io_server import PipeIOServer, terminate_process_on_failure +from tfw.components.pipe_io_server import PipeIOServer, terminate_process_on_failure LOG = logging.getLogger(__name__) DEFAULT_PERMISSIONS = 0o600 diff --git a/lib/tfw/components/process_managing_event_handler.py b/lib/tfw/builtins/process_managing_event_handler.py similarity index 100% rename from lib/tfw/components/process_managing_event_handler.py rename to lib/tfw/builtins/process_managing_event_handler.py diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/builtins/terminal_event_handler.py similarity index 100% rename from lib/tfw/components/terminal_event_handler.py rename to lib/tfw/builtins/terminal_event_handler.py diff --git a/lib/tfw/components/__init__.py b/lib/tfw/components/__init__.py index b955ad8..60bd615 100644 --- a/lib/tfw/components/__init__.py +++ b/lib/tfw/components/__init__.py @@ -1,18 +1,9 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from .directory_monitoring_event_handler import DirectoryMonitoringEventHandler -from .process_managing_event_handler import ProcessManagingEventHandler -from .terminal_event_handler import TerminalEventHandler -from .ide_event_handler import IdeEventHandler -from .history_monitor import HistoryMonitor, BashMonitor, GDBMonitor -from .terminal_commands import TerminalCommands -from .log_monitoring_event_handler import LogMonitoringEventHandler -from .fsm_managing_event_handler import FSMManagingEventHandler -from .snapshot_provider import SnapshotProvider -from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler, PipeIOServer -from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler -from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler from .commands_equal import CommandsEqual -from .frontend_event_handler import FrontendEventHandler +from .file_manager import FileManager +from .history_monitor import HistoryMonitor, BashMonitor, GDBMonitor from .message_sender import MessageSender +from .snapshot_provider import SnapshotProvider +from .terminal_commands import TerminalCommands diff --git a/lib/tfw/components/file_manager/__init__.py b/lib/tfw/components/file_manager/__init__.py new file mode 100644 index 0000000..1a24998 --- /dev/null +++ b/lib/tfw/components/file_manager/__init__.py @@ -0,0 +1 @@ +from .file_manager import FileManager diff --git a/lib/tfw/components/ide_event_handler/file_manager.py b/lib/tfw/components/file_manager/file_manager.py similarity index 100% rename from lib/tfw/components/ide_event_handler/file_manager.py rename to lib/tfw/components/file_manager/file_manager.py diff --git a/lib/tfw/components/ide_event_handler/test_file_manager.py b/lib/tfw/components/file_manager/test_file_manager.py similarity index 100% rename from lib/tfw/components/ide_event_handler/test_file_manager.py rename to lib/tfw/components/file_manager/test_file_manager.py diff --git a/lib/tfw/components/ide_event_handler/__init__.py b/lib/tfw/components/ide_event_handler/__init__.py deleted file mode 100644 index d597b12..0000000 --- a/lib/tfw/components/ide_event_handler/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .ide_event_handler import IdeEventHandler From cbc62c5a10e2920886d7b5e71404878912fe90bb Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 20 Jun 2019 16:04:51 +0200 Subject: [PATCH 035/174] Switch import order --- lib/tfw/components/inotify/test_inotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/components/inotify/test_inotify.py b/lib/tfw/components/inotify/test_inotify.py index e1e72c8..5eccd1c 100644 --- a/lib/tfw/components/inotify/test_inotify.py +++ b/lib/tfw/components/inotify/test_inotify.py @@ -8,8 +8,8 @@ from os.path import join from os import mkdir, rename from tempfile import TemporaryDirectory -import pytest import watchdog +import pytest from inotify import InotifyObserver from inotify import ( From d031f481b8a7542aad5f9b933c44a13cef0f5129 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 24 Jun 2019 14:21:30 +0200 Subject: [PATCH 036/174] Print exceptions with stack trace --- lib/tfw/config/log.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/tfw/config/log.py b/lib/tfw/config/log.py index 332fef0..827bff9 100644 --- a/lib/tfw/config/log.py +++ b/lib/tfw/config/log.py @@ -1,6 +1,8 @@ # pylint: disable=bad-whitespace from sys import stderr +from collections import deque from datetime import datetime +from traceback import format_exception, walk_tb from logging import DEBUG, getLogger, Handler, Formatter, Filter TFW_LOG_PATH = '/var/log/tfw.log' @@ -71,6 +73,10 @@ class TFWLogFormatter(Formatter): 'NOTSET' : COLOR.CYAN } + def __init__(self): + self.last_trace = None + super().__init__() + def format(self, record): date = datetime.utcfromtimestamp(record.created).strftime('%H:%M:%S') if record.args: @@ -82,10 +88,23 @@ class TFWLogFormatter(Formatter): short_message = record.msg long_message = record.msg + if record.exc_info: + current_trace = self.fetch_exception_origin(record.exc_info[2]) + if current_trace != self.last_trace: + self.last_trace = current_trace + trace = '\n'+''.join(format_exception(*record.exc_info)) + else: + trace = (f'\nSee previous traceback...\n' + f'{record.exc_info[0].__name__}: {record.exc_info[1]}') + else: + trace = '' + short_entry = (f'[{COLOR.GREY}{date}{COLOR.RESET}|>' f'{self.severity[record.levelname]}{record.module}:' - f'{record.levelname.lower()}{COLOR.RESET}] {short_message}') - long_entry = f'[{date}|>{record.module}:{record.levelname.lower()}] {long_message}' + f'{record.levelname.lower()}{COLOR.RESET}] {short_message}' + f'{trace}') + long_entry = (f'[{date}|>{record.module}:{record.levelname.lower()}] ' + f'{long_message}{trace}') return short_entry, long_entry def trim(self, value, in_dict=False): @@ -97,6 +116,10 @@ class TFWLogFormatter(Formatter): value_str = str(value) return value_str if len(value_str) <= 20 else f'{value_str[:20]}...' + @staticmethod + def fetch_exception_origin(trace): + return deque(walk_tb(trace), maxlen=1).pop() + class TFWLogWhitelistFilter(Filter): def __init__(self, names): From 65c8bcee2b780db649888be7694cbc0119f673cd Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 24 Jun 2019 14:24:59 +0200 Subject: [PATCH 037/174] Remove DirectoryMonitoringEventHandler --- lib/tfw/builtins/__init__.py | 1 - .../directory_monitoring_event_handler.py | 70 ------------------- 2 files changed, 71 deletions(-) delete mode 100644 lib/tfw/builtins/directory_monitoring_event_handler.py diff --git a/lib/tfw/builtins/__init__.py b/lib/tfw/builtins/__init__.py index 7e94f10..9061f3d 100644 --- a/lib/tfw/builtins/__init__.py +++ b/lib/tfw/builtins/__init__.py @@ -1,4 +1,3 @@ -from .directory_monitoring_event_handler import DirectoryMonitoringEventHandler from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler from .frontend_event_handler import FrontendEventHandler from .fsm_managing_event_handler import FSMManagingEventHandler diff --git a/lib/tfw/builtins/directory_monitoring_event_handler.py b/lib/tfw/builtins/directory_monitoring_event_handler.py deleted file mode 100644 index 222736c..0000000 --- a/lib/tfw/builtins/directory_monitoring_event_handler.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -import logging -from os.path import isdir, exists - -from tfw.event_handlers import FrontendEventHandlerBase -from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin -from tfw.components.directory_monitor import DirectoryMonitor - -LOG = logging.getLogger(__name__) - - -class DirectoryMonitoringEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): - def __init__(self, key, directory): - super().__init__(key) - self._directory = directory - MonitorManagerMixin.__init__( - self, - DirectoryMonitor, - key, - self._directory - ) - - self.commands = { - 'pause': self.pause, - 'resume': self.resume, - 'ignore': self.ignore, - 'selectdir': self.selectdir - } - - @property - def directory(self): - return self._directory - - @directory.setter - def directory(self, directory): - if not exists(directory) or not isdir(directory): - raise EnvironmentError('No such directory!') - self._directory = directory - - def handle_event(self, message): - try: - message['data'] = self.commands[message['data']['command']](message['data']) - return message - except KeyError: - LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) - - def pause(self, data): - self.monitor.pause() - return data - - def resume(self, data): - self.monitor.resume() - return data - - def ignore(self, data): - self.monitor.ignore += data['ignore'] - return data - - def selectdir(self, data): - try: - self.directory = data['directory'] - self.reload_monitor() - return data - except EnvironmentError: - LOG.error('Failed to switch directory!') - - def cleanup(self): - self.monitor.stop() From 597523b3b9ae8728594c23ea2f6f977767a9c1df Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 24 Jun 2019 14:27:15 +0200 Subject: [PATCH 038/174] Make observing multiple directories possible --- lib/tfw/components/inotify/inotify.py | 33 +++++++++++++--------- lib/tfw/components/inotify/test_inotify.py | 10 +++---- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/lib/tfw/components/inotify/inotify.py b/lib/tfw/components/inotify/inotify.py index 032fd67..24abb8c 100644 --- a/lib/tfw/components/inotify/inotify.py +++ b/lib/tfw/components/inotify/inotify.py @@ -1,6 +1,7 @@ # pylint: disable=too-few-public-methods from time import time +from typing import Iterable from watchdog.observers import Observer from watchdog.events import FileSystemMovedEvent, PatternMatchingEventHandler @@ -64,31 +65,37 @@ class InotifyDirDeletedEvent(InotifyEvent): class InotifyObserver: - def __init__(self, path, patterns=None, exclude=None, recursive=False): - self._path = path + def __init__(self, paths, patterns=None, exclude=None, recursive=False): + self._paths = paths self._patterns = patterns self._exclude = exclude self._recursive = recursive self._observer = Observer() - self._reset(path, patterns, exclude) + self._reset(paths, patterns, exclude) - def _reset(self, path, patterns, exclude): + def _reset(self, paths, patterns, exclude): dispatch_event = self.dispatch_event class TransformerEventHandler(PatternMatchingEventHandler): def on_any_event(self, event): dispatch_event(event) self.handler = TransformerEventHandler(patterns, exclude) self._observer.unschedule_all() - self._observer.schedule(self.handler, path, self._recursive) + if isinstance(paths, str): + self._observer.schedule(self.handler, paths, self._recursive) + elif isinstance(paths, Iterable): + for path in paths: + self._observer.schedule(self.handler, path, self._recursive) + else: + raise ValueError('Expected one or more strings representing paths.') @property - def path(self): - return self._path + def paths(self): + return self._paths - @path.setter - def path(self, path): - self._path = path - self._reset(path, self._patterns, self._exclude) + @paths.setter + def paths(self, paths): + self._paths = paths + self._reset(paths, self._patterns, self._exclude) @property def patterns(self): @@ -97,7 +104,7 @@ class InotifyObserver: @patterns.setter def patterns(self, patterns): self._patterns = patterns - self._reset(self._path, patterns, self._exclude) + self._reset(self._paths, patterns, self._exclude) @property def exclude(self): @@ -106,7 +113,7 @@ class InotifyObserver: @exclude.setter def exclude(self, exclude): self._exclude = exclude - self._reset(self._path, self._patterns, exclude) + self._reset(self._paths, self._patterns, exclude) def start(self): self._observer.start() diff --git a/lib/tfw/components/inotify/test_inotify.py b/lib/tfw/components/inotify/test_inotify.py index 5eccd1c..d0ac637 100644 --- a/lib/tfw/components/inotify/test_inotify.py +++ b/lib/tfw/components/inotify/test_inotify.py @@ -71,10 +71,10 @@ class InotifyContext: class InotifyTestObserver(InotifyObserver): - def __init__(self, path, patterns=None, exclude=None, recursive=False): + def __init__(self, paths, patterns=None, exclude=None, recursive=False): self.any_list = [] self.create_queue, self.modify_queue, self.move_queue, self.delete_queue = [Queue() for _ in range(4)] - super().__init__(path, patterns, exclude, recursive) + super().__init__(paths, patterns, exclude, recursive) def on_any_event(self, event): self.any_list.append(event) @@ -138,12 +138,12 @@ def test_delete(context): context.check_event(InotifyDirDeletedEvent, context.subdir) assert context.check_any() -def test_path(context): - context.observer.path = context.subdir +def test_paths(context): + context.observer.paths = context.subdir context.create_random_folder(context.workdir) newfile = context.create_random_file(context.subdir, '.txt') context.check_event(InotifyFileCreatedEvent, newfile) - context.observer.path = context.subdir + context.observer.paths = context.subdir assert context.check_any() def test_patterns(context): From 494761d2a76ada2f5265df653944acb80651660c Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 24 Jun 2019 14:29:31 +0200 Subject: [PATCH 039/174] Replace mixins with inotify based observers --- lib/tfw/builtins/ide_event_handler.py | 35 ++++---- .../builtins/log_monitoring_event_handler.py | 64 ++++++++++++--- .../process_managing_event_handler.py | 5 +- lib/tfw/components/directory_monitor.py | 81 ------------------- lib/tfw/components/log_monitor.py | 58 ------------- lib/tfw/mixins/monitor_manager_mixin.py | 30 ------- lib/tfw/mixins/observer_mixin.py | 20 ----- 7 files changed, 71 insertions(+), 222 deletions(-) delete mode 100644 lib/tfw/components/directory_monitor.py delete mode 100644 lib/tfw/components/log_monitor.py delete mode 100644 lib/tfw/mixins/monitor_manager_mixin.py delete mode 100644 lib/tfw/mixins/observer_mixin.py diff --git a/lib/tfw/builtins/ide_event_handler.py b/lib/tfw/builtins/ide_event_handler.py index e91213b..2338a8e 100644 --- a/lib/tfw/builtins/ide_event_handler.py +++ b/lib/tfw/builtins/ide_event_handler.py @@ -2,20 +2,29 @@ # All Rights Reserved. See LICENSE file for details. import logging -from os.path import isfile, join, relpath, exists, isdir, realpath -from glob import glob -from fnmatch import fnmatchcase -from typing import Iterable -from tfw.event_handlers import FrontendEventHandlerBase -from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin +from tfw.networking import Scope +from tfw.event_handlers import FrontendEventHandlerBase, TFWServerUplinkConnector from tfw.components import FileManager -from tfw.components.directory_monitor import DirectoryMonitor +from tfw.components.inotify import InotifyObserver LOG = logging.getLogger(__name__) -class IdeEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): +class IdeInotifyObserver(InotifyObserver): + def __init__(self, paths): + self.uplink = TFWServerUplinkConnector() + super().__init__(paths) + + def on_modified(self, event): + LOG.debug(event) + self.uplink.send_message({ + 'key': 'ide', + 'data': {'command': 'reload'} + }, Scope.WEBSOCKET) + + +class IdeEventHandler(FrontendEventHandlerBase): # pylint: disable=too-many-arguments,anomalous-backslash-in-string """ Event handler implementing the backend of our browser based IDE. @@ -51,12 +60,8 @@ class IdeEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): f'No file(s) in IdeEventHandler working_directory "{directory}"!' ) - MonitorManagerMixin.__init__( - self, - DirectoryMonitor, - self.key, - self.filemanager.allowed_directories - ) + self.monitor = IdeInotifyObserver(self.filemanager.allowed_directories) + self.monitor.start() self.commands = { 'read': self.read, @@ -91,7 +96,6 @@ class IdeEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): (new file content) """ - self.monitor.ignore = self.monitor.ignore + 1 try: self.filemanager.file_contents = data['content'] except Exception: # pylint: disable=broad-except @@ -123,7 +127,6 @@ class IdeEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): """ try: self.filemanager.workdir = data['directory'] - self.reload_monitor() try: self.filemanager.filename = self.filemanager.files[0] self.read(data) diff --git a/lib/tfw/builtins/log_monitoring_event_handler.py b/lib/tfw/builtins/log_monitoring_event_handler.py index e324786..42a4f79 100644 --- a/lib/tfw/builtins/log_monitoring_event_handler.py +++ b/lib/tfw/builtins/log_monitoring_event_handler.py @@ -3,14 +3,57 @@ import logging -from tfw.event_handlers import FrontendEventHandlerBase -from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin -from tfw.components.log_monitor import LogMonitor +from os.path import dirname +from tfw.networking import Scope +from tfw.event_handlers import FrontendEventHandlerBase, TFWServerUplinkConnector +from tfw.components.inotify import InotifyObserver +from tfw.mixins.supervisor_mixin import SupervisorLogMixin LOG = logging.getLogger(__name__) -class LogMonitoringEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): +class LogInotifyObserver(InotifyObserver, SupervisorLogMixin): + def __init__(self, process_name, log_tail=0): + self.prevent_log_recursion() + self.uplink = TFWServerUplinkConnector() + self.process_name = process_name + self.log_tail = log_tail + self.procinfo = self.supervisor.getProcessInfo(self.process_name) + super().__init__( + [dirname(self.procinfo['stdout_logfile']), dirname(self.procinfo['stderr_logfile'])], + [self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']] + ) + + def reset(self, process_name, log_tail): + self.process_name = process_name + self.log_tail = log_tail + self.procinfo = self.supervisor.getProcessInfo(self.process_name) + self.paths = [ + dirname(self.procinfo['stdout_logfile']), + dirname(self.procinfo['stderr_logfile']) + ] + self.patterns = [ + self.procinfo['stdout_logfile'], + self.procinfo['stderr_logfile'] + ] + + def on_modified(self, event): + self.uplink.send_message({ + 'key': 'processlog', + 'data': { + 'command': 'new_log', + 'stdout': self.read_stdout(self.process_name, tail=self.log_tail), + 'stderr': self.read_stderr(self.process_name, tail=self.log_tail) + } + }, Scope.BROADCAST) + + @staticmethod + def prevent_log_recursion(): + # This is done to prevent inotify event logs triggering themselves (infinite log recursion) + logging.getLogger('watchdog.observers.inotify_buffer').propagate = False + + +class LogMonitoringEventHandler(FrontendEventHandlerBase): """ Monitors the output of a supervisor process (stdout, stderr) and sends the results to the frontend. @@ -24,12 +67,8 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): super().__init__(key) self.process_name = process_name self.log_tail = log_tail - MonitorManagerMixin.__init__( - self, - LogMonitor, - self.process_name, - self.log_tail - ) + self.monitor = LogInotifyObserver(process_name, log_tail) + self.monitor.start() self.command_handlers = { 'process_name': self.handle_process_name, @@ -40,7 +79,6 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): try: data = message['data'] self.command_handlers[data['command']](data) - self.reload_monitor() except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) @@ -51,7 +89,7 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): :param data: TFW message data containing 'value' (name of the process to monitor) """ - self.set_monitor_args(data['value'], self.log_tail) + self.monitor.reset(data['value'], self.log_tail) def handle_log_tail(self, data): """ @@ -62,7 +100,7 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase, MonitorManagerMixin): :param data: TFW message data containing 'value' (new tail length) """ - self.set_monitor_args(self.process_name, data['value']) + self.monitor.reset(self.process_name, data['value']) def cleanup(self): self.monitor.stop() diff --git a/lib/tfw/builtins/process_managing_event_handler.py b/lib/tfw/builtins/process_managing_event_handler.py index 37141d0..4c7daaa 100644 --- a/lib/tfw/builtins/process_managing_event_handler.py +++ b/lib/tfw/builtins/process_managing_event_handler.py @@ -6,7 +6,6 @@ from xmlrpc.client import Fault as SupervisorFault from tfw.event_handlers import FrontendEventHandlerBase from tfw.mixins.supervisor_mixin import SupervisorMixin, SupervisorLogMixin -from tfw.components.directory_monitor import with_monitor_paused LOG = logging.getLogger(__name__) @@ -36,13 +35,11 @@ class ProcessManagingEventHandler(FrontendEventHandlerBase): Commands available: start, stop, restart, readlog (the names are as self-documenting as it gets) """ - def __init__(self, key, dirmonitor=None, log_tail=0): + def __init__(self, key, log_tail=0): super().__init__(key) - self.monitor = dirmonitor self.processmanager = ProcessManager() self.log_tail = log_tail - @with_monitor_paused def handle_event(self, message): try: data = message['data'] diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py deleted file mode 100644 index 65dc275..0000000 --- a/lib/tfw/components/directory_monitor.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -import logging -from functools import wraps - -from watchdog.events import FileSystemEventHandler as FileSystemWatchdogEventHandler - -from tfw.networking import Scope -from tfw.event_handlers import TFWServerUplinkConnector -from tfw.decorators.rate_limiter import RateLimiter -from tfw.mixins.observer_mixin import ObserverMixin - -LOG = logging.getLogger(__name__) - - -class DirectoryMonitor(ObserverMixin): - def __init__(self, ide_key, directories): - self.eventhandler = IdeReloadWatchdogEventHandler(ide_key) - for directory in directories: - self.observer.schedule(self.eventhandler, directory, recursive=True) - - self.pause, self.resume = self.eventhandler.pause, self.eventhandler.resume - - @property - def ignore(self): - return self.eventhandler.ignore - - @ignore.setter - def ignore(self, value): - self.eventhandler.ignore = value if value >= 0 else 0 - - @property - def pauser(self): - return DirectoryMonitor.Pauser(self) - - class Pauser: - def __init__(self, directory_monitor): - self.directorymonitor = directory_monitor - def __enter__(self): - self.directorymonitor.pause() - def __exit__(self, exc_type, exc_val, exc_tb): - self.directorymonitor.resume() - - -class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler): - def __init__(self, ide_key): - super().__init__() - self.ide_key = ide_key - self.uplink = TFWServerUplinkConnector() - self._paused = False - self.ignore = 0 - - def pause(self): - self._paused = True - - def resume(self): - self._paused = False - - @RateLimiter(rate_per_second=2) - def on_modified(self, event): - if self._paused: - return - if self.ignore > 0: - self.ignore = self.ignore - 1 - return - LOG.debug(event) - self.uplink.send_message({ - 'key': self.ide_key, - 'data': {'command': 'reload'} - }, Scope.WEBSOCKET) - - -def with_monitor_paused(fun): - @wraps(fun) - def wrapper(self, *args, **kwargs): - if self.monitor: - with self.monitor.pauser: - return fun(self, *args, **kwargs) - return fun(self, *args, **kwargs) - return wrapper diff --git a/lib/tfw/components/log_monitor.py b/lib/tfw/components/log_monitor.py deleted file mode 100644 index 14591e8..0000000 --- a/lib/tfw/components/log_monitor.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -import logging -from os.path import dirname - -from watchdog.events import PatternMatchingEventHandler as PatternMatchingWatchdogEventHandler - -from tfw.networking import Scope -from tfw.event_handlers import TFWServerUplinkConnector -from tfw.decorators.rate_limiter import RateLimiter -from tfw.mixins.observer_mixin import ObserverMixin -from tfw.mixins.supervisor_mixin import SupervisorLogMixin - - -class LogMonitor(ObserverMixin): - def __init__(self, process_name, log_tail=0): - self.prevent_log_recursion() - event_handler = SendLogWatchdogEventHandler( - process_name, - log_tail=log_tail - ) - self.observer.schedule( - event_handler, - event_handler.path - ) - - @staticmethod - def prevent_log_recursion(): - # This is done to prevent inotify event logs triggering themselves (infinite log recursion) - logging.getLogger('watchdog.observers.inotify_buffer').propagate = False - - -class SendLogWatchdogEventHandler(PatternMatchingWatchdogEventHandler, SupervisorLogMixin): - def __init__(self, process_name, log_tail=0): - self.process_name = process_name - self.procinfo = self.supervisor.getProcessInfo(self.process_name) - super().__init__([ - self.procinfo['stdout_logfile'], - self.procinfo['stderr_logfile'] - ]) - self.uplink = TFWServerUplinkConnector() - self.log_tail = log_tail - - @property - def path(self): - return dirname(self.procinfo['stdout_logfile']) - - @RateLimiter(rate_per_second=5) - def on_modified(self, event): - self.uplink.send_message({ - 'key': 'processlog', - 'data': { - 'command': 'new_log', - 'stdout': self.read_stdout(self.process_name, tail=self.log_tail), - 'stderr': self.read_stderr(self.process_name, tail=self.log_tail) - } - }, Scope.BROADCAST) diff --git a/lib/tfw/mixins/monitor_manager_mixin.py b/lib/tfw/mixins/monitor_manager_mixin.py deleted file mode 100644 index c153ff9..0000000 --- a/lib/tfw/mixins/monitor_manager_mixin.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -import logging - -LOG = logging.getLogger(__name__) - - -class MonitorManagerMixin: - def __init__(self, monitor_type, *monitor_args): - self._monitor_type = monitor_type - self._monitor = None - self.monitor_args = monitor_args - self.reload_monitor() - - @property - def monitor(self): - return self._monitor - - def set_monitor_args(self, *monitor_args): - self.monitor_args = monitor_args - - def reload_monitor(self): - if self._monitor: - try: - self._monitor.stop() - except KeyError: - LOG.debug('Working directory was removed – ignoring...') - self._monitor = self._monitor_type(*self.monitor_args) - self._monitor.watch() # This runs on a separate thread diff --git a/lib/tfw/mixins/observer_mixin.py b/lib/tfw/mixins/observer_mixin.py deleted file mode 100644 index d5415e5..0000000 --- a/lib/tfw/mixins/observer_mixin.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -from watchdog.observers import Observer - -from tfw.decorators.lazy_property import lazy_property - - -class ObserverMixin: - @lazy_property - def observer(self): - # pylint: disable=no-self-use - return Observer() - - def watch(self): - self.observer.start() - - def stop(self): - self.observer.stop() - self.observer.join() From d8e2f8a92ab43009f296642b38c9c0794b8b5c82 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 25 Jun 2019 11:31:27 +0200 Subject: [PATCH 040/174] Make file operations unbuffered --- lib/tfw/components/file_manager/file_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/tfw/components/file_manager/file_manager.py b/lib/tfw/components/file_manager/file_manager.py index 967e06f..0e4fc6a 100644 --- a/lib/tfw/components/file_manager/file_manager.py +++ b/lib/tfw/components/file_manager/file_manager.py @@ -65,13 +65,13 @@ class FileManager: # pylint: disable=too-many-instance-attributes @property def file_contents(self): - with open(self._filepath(self.filename), 'r', errors='surrogateescape') as ifile: - return ifile.read() + with open(self._filepath(self.filename), 'rb', buffering=0) as ifile: + return ifile.read().decode(errors='surrogateescape') @file_contents.setter def file_contents(self, value): - with open(self._filepath(self.filename), 'w', errors='surrogateescape') as ofile: - ofile.write(value) + with open(self._filepath(self.filename), 'wb', buffering=0) as ofile: + ofile.write(value.encode()) def _is_in_allowed_dir(self, path): return any( From 5e06f72c35ea8219f79b790b833da5b87b6a8d11 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 25 Jun 2019 11:32:11 +0200 Subject: [PATCH 041/174] Rename private methods to be underscored --- lib/tfw/components/inotify/inotify.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/tfw/components/inotify/inotify.py b/lib/tfw/components/inotify/inotify.py index 24abb8c..80b7902 100644 --- a/lib/tfw/components/inotify/inotify.py +++ b/lib/tfw/components/inotify/inotify.py @@ -74,10 +74,10 @@ class InotifyObserver: self._reset(paths, patterns, exclude) def _reset(self, paths, patterns, exclude): - dispatch_event = self.dispatch_event + _dispatch_event = self._dispatch_event class TransformerEventHandler(PatternMatchingEventHandler): def on_any_event(self, event): - dispatch_event(event) + _dispatch_event(event) self.handler = TransformerEventHandler(patterns, exclude) self._observer.unschedule_all() if isinstance(paths, str): @@ -122,7 +122,7 @@ class InotifyObserver: self._observer.stop() self._observer.join() - def dispatch_event(self, event): + def _dispatch_event(self, event): event_to_action = { InotifyFileCreatedEvent : self.on_created, InotifyFileModifiedEvent : self.on_modified, @@ -134,12 +134,12 @@ class InotifyObserver: InotifyDirDeletedEvent : self.on_deleted } - event = self.transform_event(event) + event = self._transform_event(event) self.on_any_event(event) event_to_action[type(event)](event) @staticmethod - def transform_event(event): + def _transform_event(event): watchdog_to_inotify = { FileCreatedEvent : InotifyFileCreatedEvent, FileModifiedEvent : InotifyFileModifiedEvent, From b9ee2031e2345a06206809507d0d249e8ac69102 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 25 Jun 2019 11:39:55 +0200 Subject: [PATCH 042/174] Delete unused import --- lib/tfw/builtins/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/builtins/__init__.py b/lib/tfw/builtins/__init__.py index 9061f3d..129296d 100644 --- a/lib/tfw/builtins/__init__.py +++ b/lib/tfw/builtins/__init__.py @@ -3,7 +3,7 @@ from .frontend_event_handler import FrontendEventHandler from .fsm_managing_event_handler import FSMManagingEventHandler from .ide_event_handler import IdeEventHandler from .log_monitoring_event_handler import LogMonitoringEventHandler -from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler, PipeIOServer +from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler from .process_managing_event_handler import ProcessManagingEventHandler from .terminal_event_handler import TerminalEventHandler From 149dcb0b4b6548a41436717f99bca7796740d8e0 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 27 Jun 2019 14:04:35 +0200 Subject: [PATCH 043/174] Move MessageSender due to dependencies --- lib/tfw/builtins/__init__.py | 1 + lib/tfw/builtins/frontend_event_handler.py | 2 +- lib/tfw/{components => builtins}/message_sender.py | 0 lib/tfw/components/__init__.py | 1 - 4 files changed, 2 insertions(+), 2 deletions(-) rename lib/tfw/{components => builtins}/message_sender.py (100%) diff --git a/lib/tfw/builtins/__init__.py b/lib/tfw/builtins/__init__.py index 129296d..c554666 100644 --- a/lib/tfw/builtins/__init__.py +++ b/lib/tfw/builtins/__init__.py @@ -3,6 +3,7 @@ from .frontend_event_handler import FrontendEventHandler from .fsm_managing_event_handler import FSMManagingEventHandler from .ide_event_handler import IdeEventHandler from .log_monitoring_event_handler import LogMonitoringEventHandler +from .message_sender import MessageSender from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler from .process_managing_event_handler import ProcessManagingEventHandler diff --git a/lib/tfw/builtins/frontend_event_handler.py b/lib/tfw/builtins/frontend_event_handler.py index b372023..d3fe91e 100644 --- a/lib/tfw/builtins/frontend_event_handler.py +++ b/lib/tfw/builtins/frontend_event_handler.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from contextlib import suppress -from tfw.components.message_sender import MessageSender +from tfw.builtins.message_sender import MessageSender from tfw.event_handlers import FrontendEventHandlerBase diff --git a/lib/tfw/components/message_sender.py b/lib/tfw/builtins/message_sender.py similarity index 100% rename from lib/tfw/components/message_sender.py rename to lib/tfw/builtins/message_sender.py diff --git a/lib/tfw/components/__init__.py b/lib/tfw/components/__init__.py index 60bd615..203f7e1 100644 --- a/lib/tfw/components/__init__.py +++ b/lib/tfw/components/__init__.py @@ -4,6 +4,5 @@ from .commands_equal import CommandsEqual from .file_manager import FileManager from .history_monitor import HistoryMonitor, BashMonitor, GDBMonitor -from .message_sender import MessageSender from .snapshot_provider import SnapshotProvider from .terminal_commands import TerminalCommands From 778f15558079619818f26e4cdaa742b8f3efb482 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 27 Jun 2019 14:08:39 +0200 Subject: [PATCH 044/174] Acquire log path from the environment --- Dockerfile | 1 + lib/tfw/config/log.py | 22 ++++++++++------------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 94eee57..7bfe6e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,6 +42,7 @@ ENV PYTHONPATH="/usr/local/lib" \ TFW_SNAPSHOTS_DIR="/.tfw/snapshots" \ TFW_AUTH_KEY="/tmp/tfw-auth.key" \ TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history" \ + TFW_LOGFILE="/var/log/tfw.log" \ PROMPT_COMMAND="history -a" COPY bashrc /tmp diff --git a/lib/tfw/config/log.py b/lib/tfw/config/log.py index 827bff9..6c85aa7 100644 --- a/lib/tfw/config/log.py +++ b/lib/tfw/config/log.py @@ -5,7 +5,7 @@ from datetime import datetime from traceback import format_exception, walk_tb from logging import DEBUG, getLogger, Handler, Formatter, Filter -TFW_LOG_PATH = '/var/log/tfw.log' +from .envvars import TFWENV class COLOR: @@ -29,20 +29,18 @@ class COLOR: class TFWLog: - def __init__(self, path=TFW_LOG_PATH, level=DEBUG): + def __init__(self, path=TFWENV.LOGFILE, level=DEBUG): self.log = getLogger() self.old_level = self.log.level self.new_level = level self.handler = TFWLogHandler(path) - self.handler.setFormatter(TFWLogFormatter()) + self.handler.setFormatter(TFWLogFormatter(20)) def start(self): self.log.setLevel(self.new_level) self.log.addHandler(self.handler) - self.log.info('Logging started.') def stop(self): - self.log.info('Stop logging.') self.log.setLevel(self.old_level) self.handler.close() self.log.removeHandler(self.handler) @@ -64,7 +62,7 @@ class TFWLogHandler(Handler): self.logfile.close() class TFWLogFormatter(Formatter): - severity = { + severity_to_color = { 'CRITICAL' : COLOR.BOLDRED, 'ERROR' : COLOR.RED, 'WARNING' : COLOR.YELLOW, @@ -73,7 +71,8 @@ class TFWLogFormatter(Formatter): 'NOTSET' : COLOR.CYAN } - def __init__(self): + def __init__(self, limit): + self.limit = limit self.last_trace = None super().__init__() @@ -100,21 +99,20 @@ class TFWLogFormatter(Formatter): trace = '' short_entry = (f'[{COLOR.GREY}{date}{COLOR.RESET}|>' - f'{self.severity[record.levelname]}{record.module}:' + f'{self.severity_to_color[record.levelname]}{record.module}:' f'{record.levelname.lower()}{COLOR.RESET}] {short_message}' f'{trace}') long_entry = (f'[{date}|>{record.module}:{record.levelname.lower()}] ' f'{long_message}{trace}') return short_entry, long_entry - def trim(self, value, in_dict=False): + def trim(self, value): if isinstance(value, dict): - trimmed = {k: self.trim(v, True) for k, v in value.items()} - return trimmed if in_dict else str(trimmed) + return {k: self.trim(v, True) for k, v in value.items()} if isinstance(value, (int, float)): return value value_str = str(value) - return value_str if len(value_str) <= 20 else f'{value_str[:20]}...' + return value_str if len(value_str) <= self.limit else f'{value_str[:self.limit]}...' @staticmethod def fetch_exception_origin(trace): From ffe512776aa9f63cd421143ab9210dfb3017ca33 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 27 Jun 2019 14:12:01 +0200 Subject: [PATCH 045/174] Make files observable and refactor event handlers --- lib/tfw/builtins/ide_event_handler.py | 43 ++++++++++----- .../builtins/log_monitoring_event_handler.py | 40 ++++++-------- lib/tfw/builtins/terminal_event_handler.py | 11 ++-- lib/tfw/components/history_monitor.py | 49 +++++++++++------ lib/tfw/components/inotify/inotify.py | 53 ++++++++++++------- lib/tfw/components/inotify/test_inotify.py | 8 +-- 6 files changed, 123 insertions(+), 81 deletions(-) diff --git a/lib/tfw/builtins/ide_event_handler.py b/lib/tfw/builtins/ide_event_handler.py index 2338a8e..f4025b0 100644 --- a/lib/tfw/builtins/ide_event_handler.py +++ b/lib/tfw/builtins/ide_event_handler.py @@ -10,18 +10,28 @@ from tfw.components.inotify import InotifyObserver LOG = logging.getLogger(__name__) - -class IdeInotifyObserver(InotifyObserver): - def __init__(self, paths): - self.uplink = TFWServerUplinkConnector() - super().__init__(paths) - - def on_modified(self, event): - LOG.debug(event) - self.uplink.send_message({ - 'key': 'ide', - 'data': {'command': 'reload'} - }, Scope.WEBSOCKET) +BUILD_ARTIFACTS = [ + "*.a", + "*.class", + "*.dll", + "*.dylib", + "*.elf", + "*.exe", + "*.jar", + "*.ko", + "*.la", + "*.lib", + "*.lo", + "*.o", + "*.obj", + "*.out", + "*.py[cod]", + "*.so", + "*.so.*", + "*.tar.gz", + "*.zip", + "*__pycache__*" +] class IdeEventHandler(FrontendEventHandlerBase): @@ -60,7 +70,14 @@ class IdeEventHandler(FrontendEventHandlerBase): f'No file(s) in IdeEventHandler working_directory "{directory}"!' ) - self.monitor = IdeInotifyObserver(self.filemanager.allowed_directories) + self.monitor = InotifyObserver(self.filemanager.allowed_directories, exclude=BUILD_ARTIFACTS) + def on_modified(event): + LOG.debug(event) + self.server_connector.send_message({ + 'key': 'ide', + 'data': {'command': 'reload'} + }, Scope.WEBSOCKET) + self.monitor.on_modified = on_modified self.monitor.start() self.commands = { diff --git a/lib/tfw/builtins/log_monitoring_event_handler.py b/lib/tfw/builtins/log_monitoring_event_handler.py index 42a4f79..f1d2919 100644 --- a/lib/tfw/builtins/log_monitoring_event_handler.py +++ b/lib/tfw/builtins/log_monitoring_event_handler.py @@ -13,32 +13,26 @@ LOG = logging.getLogger(__name__) class LogInotifyObserver(InotifyObserver, SupervisorLogMixin): - def __init__(self, process_name, log_tail=0): + def __init__(self, server_connector, process_name, log_tail=0): self.prevent_log_recursion() - self.uplink = TFWServerUplinkConnector() - self.process_name = process_name + self.server_connector = server_connector + self._process_name = process_name self.log_tail = log_tail self.procinfo = self.supervisor.getProcessInfo(self.process_name) - super().__init__( - [dirname(self.procinfo['stdout_logfile']), dirname(self.procinfo['stderr_logfile'])], - [self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']] - ) + InotifyObserver.__init__(self, [self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']]) - def reset(self, process_name, log_tail): - self.process_name = process_name - self.log_tail = log_tail - self.procinfo = self.supervisor.getProcessInfo(self.process_name) - self.paths = [ - dirname(self.procinfo['stdout_logfile']), - dirname(self.procinfo['stderr_logfile']) - ] - self.patterns = [ - self.procinfo['stdout_logfile'], - self.procinfo['stderr_logfile'] - ] + @property + def process_name(self): + return self._process_name + + @process_name.setter + def process_name(self, process_name): + self._process_name = process_name + self.procinfo = self.supervisor.getProcessInfo(process_name) + self.paths = [self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']] def on_modified(self, event): - self.uplink.send_message({ + self.server_connector.send_message({ 'key': 'processlog', 'data': { 'command': 'new_log', @@ -67,7 +61,7 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase): super().__init__(key) self.process_name = process_name self.log_tail = log_tail - self.monitor = LogInotifyObserver(process_name, log_tail) + self.monitor = LogInotifyObserver(self.server_connector, process_name, log_tail) self.monitor.start() self.command_handlers = { @@ -89,7 +83,7 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase): :param data: TFW message data containing 'value' (name of the process to monitor) """ - self.monitor.reset(data['value'], self.log_tail) + self.monitor.process_name = data['value'] def handle_log_tail(self, data): """ @@ -100,7 +94,7 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase): :param data: TFW message data containing 'value' (new tail length) """ - self.monitor.reset(self.process_name, data['value']) + self.monitor.log_tail = data['value'] def cleanup(self): self.monitor.stop() diff --git a/lib/tfw/builtins/terminal_event_handler.py b/lib/tfw/builtins/terminal_event_handler.py index c7104e0..6d2a14a 100644 --- a/lib/tfw/builtins/terminal_event_handler.py +++ b/lib/tfw/builtins/terminal_event_handler.py @@ -4,6 +4,7 @@ import logging from tfw.event_handlers import FrontendEventHandlerBase +from tfw.components import BashMonitor from tfw.components.terminado_mini_server import TerminadoMiniServer from tfw.config import TFWENV from tao.config import TAOENV @@ -21,13 +22,13 @@ class TerminalEventHandler(FrontendEventHandlerBase): a command to be executed. The API of each command is documented in their respective handler. """ - def __init__(self, key, monitor): + def __init__(self, key): """ :param key: key this EventHandler listens to :param monitor: tfw.components.HistoryMonitor instance to read command history from """ super().__init__(key) - self._historymonitor = monitor + self._historymonitor = BashMonitor(self.server_connector, TFWENV.HISTFILE) bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash'] self.terminado_server = TerminadoMiniServer( @@ -42,8 +43,7 @@ class TerminalEventHandler(FrontendEventHandlerBase): 'read': self.read } - if self._historymonitor: - self._historymonitor.start() + self._historymonitor.start() self.terminado_server.listen() @property @@ -84,5 +84,4 @@ class TerminalEventHandler(FrontendEventHandlerBase): def cleanup(self): self.terminado_server.stop() - if self.historymonitor: - self.historymonitor.stop() + self.historymonitor.stop() diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index a11583a..703a68f 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -1,6 +1,10 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +from pwd import getpwnam +from grp import getgrnam +from pathlib import Path +from os import chown from os.path import dirname from re import findall from re import compile as compileregex @@ -22,30 +26,39 @@ class HistoryMonitor(ABC, InotifyObserver): command pattern property and optionally the sanitize_command method. See examples below. """ - def __init__(self, domain, histfile): - self.domain = domain + def __init__(self, uplink, histfile): + self._domain = '' self.histfile = histfile - self._history = [] - self._last_length = len(self._history) - self.uplink = TFWServerUplinkConnector() - super().__init__(dirname(self.histfile), [self.histfile]) + self.history = [] + self._last_length = len(self.history) + self.uplink = uplink + uid = getpwnam('user').pw_uid + gid = getgrnam('users').gr_gid + path = Path(self.histfile) + path.touch() + chown(self.histfile, uid, gid) + super().__init__(self.histfile) + + @property + def domain(self): + return self._domain + + @domain.setter + def domain(self, domain): + self._domain = domain def on_modified(self, event): self._fetch_history() - if self._last_length < len(self._history): - for command in self._history[self._last_length:]: + if self._last_length < len(self.history): + for command in self.history[self._last_length:]: self.send_message(command) - @property - def history(self): - return self._history - def _fetch_history(self): - self._last_length = len(self._history) + self._last_length = len(self.history) with open(self.histfile, 'r') as ifile: pattern = compileregex(self.command_pattern) data = ifile.read() - self._history = [ + self.history = [ self.sanitize_command(command) for command in findall(pattern, data) ] @@ -76,8 +89,9 @@ class BashMonitor(HistoryMonitor): shopt -s histappend unset HISTCONTROL """ - def __init__(self, histfile): - super().__init__('bash', histfile) + def __init__(self, uplink, histfile): + super().__init__(uplink, histfile) + self.domain = 'bash' @property def command_pattern(self): @@ -93,7 +107,8 @@ class GDBMonitor(HistoryMonitor): For this to work "set trace-commands on" must be set in GDB. """ def __init__(self, histfile): - super().__init__('gdb', histfile) + super().__init__(histfile) + self.domain = 'gdb' @property def command_pattern(self): diff --git a/lib/tfw/components/inotify/inotify.py b/lib/tfw/components/inotify/inotify.py index 80b7902..3bbefee 100644 --- a/lib/tfw/components/inotify/inotify.py +++ b/lib/tfw/components/inotify/inotify.py @@ -1,7 +1,8 @@ # pylint: disable=too-few-public-methods -from time import time from typing import Iterable +from time import time +from os.path import abspath, dirname, isfile from watchdog.observers import Observer from watchdog.events import FileSystemMovedEvent, PatternMatchingEventHandler @@ -65,28 +66,44 @@ class InotifyDirDeletedEvent(InotifyEvent): class InotifyObserver: - def __init__(self, paths, patterns=None, exclude=None, recursive=False): - self._paths = paths + def __init__(self, path, patterns=[], exclude=None, recursive=False): + self._files = [] + self._paths = path self._patterns = patterns self._exclude = exclude self._recursive = recursive self._observer = Observer() - self._reset(paths, patterns, exclude) + self._reset() - def _reset(self, paths, patterns, exclude): - _dispatch_event = self._dispatch_event + def _reset(self): + dispatch_event = self._dispatch_event class TransformerEventHandler(PatternMatchingEventHandler): def on_any_event(self, event): - _dispatch_event(event) - self.handler = TransformerEventHandler(patterns, exclude) - self._observer.unschedule_all() - if isinstance(paths, str): - self._observer.schedule(self.handler, paths, self._recursive) - elif isinstance(paths, Iterable): - for path in paths: - self._observer.schedule(self.handler, path, self._recursive) + dispatch_event(event) + + if isinstance(self._paths, str): + self._paths = [self._paths] + if isinstance(self._paths, Iterable): + self._extract_files_from_paths() else: - raise ValueError('Expected one or more strings representing paths.') + raise ValueError('Expected one or more string paths.') + + patterns = self._files+self.patterns + self.handler = TransformerEventHandler(patterns if patterns else None, self.exclude) + self._observer.unschedule_all() + for path in self.paths: + self._observer.schedule(self.handler, path, self._recursive) + + def _extract_files_from_paths(self): + files, paths = [], [] + for path in self._paths: + if isfile(path): + new_file = abspath(path) + files.append(new_file) + paths.append(dirname(new_file)) + else: + paths.append(path) + self._files, self._paths = files, paths @property def paths(self): @@ -95,7 +112,7 @@ class InotifyObserver: @paths.setter def paths(self, paths): self._paths = paths - self._reset(paths, self._patterns, self._exclude) + self._reset() @property def patterns(self): @@ -104,7 +121,7 @@ class InotifyObserver: @patterns.setter def patterns(self, patterns): self._patterns = patterns - self._reset(self._paths, patterns, self._exclude) + self._reset() @property def exclude(self): @@ -113,7 +130,7 @@ class InotifyObserver: @exclude.setter def exclude(self, exclude): self._exclude = exclude - self._reset(self._paths, self._patterns, exclude) + self._reset() def start(self): self._observer.start() diff --git a/lib/tfw/components/inotify/test_inotify.py b/lib/tfw/components/inotify/test_inotify.py index d0ac637..c6d0e12 100644 --- a/lib/tfw/components/inotify/test_inotify.py +++ b/lib/tfw/components/inotify/test_inotify.py @@ -11,8 +11,8 @@ from tempfile import TemporaryDirectory import watchdog import pytest -from inotify import InotifyObserver -from inotify import ( +from .inotify import InotifyObserver +from .inotify import ( InotifyFileCreatedEvent, InotifyFileModifiedEvent, InotifyFileMovedEvent, InotifyFileDeletedEvent, InotifyDirCreatedEvent, InotifyDirModifiedEvent, InotifyDirMovedEvent, InotifyDirDeletedEvent @@ -113,8 +113,8 @@ def test_create(context): assert context.check_any() def test_modify(context): - with open(context.subfile, 'w') as ofile: - ofile.write('text') + with open(context.subfile, 'wb', buffering=0) as ofile: + ofile.write(b'text') context.check_event(InotifyFileModifiedEvent, context.subfile) while True: try: From 810f6d529730f479f50c4972053eef9a9cb83dba Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 27 Jun 2019 14:17:57 +0200 Subject: [PATCH 046/174] Remove deprecated argument --- lib/tfw/config/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/config/log.py b/lib/tfw/config/log.py index 6c85aa7..c722cf4 100644 --- a/lib/tfw/config/log.py +++ b/lib/tfw/config/log.py @@ -108,7 +108,7 @@ class TFWLogFormatter(Formatter): def trim(self, value): if isinstance(value, dict): - return {k: self.trim(v, True) for k, v in value.items()} + return {k: self.trim(v) for k, v in value.items()} if isinstance(value, (int, float)): return value value_str = str(value) From 28f09086a2fef8515acf9ba411db0b4a6496f7bd Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 27 Jun 2019 14:36:18 +0200 Subject: [PATCH 047/174] Fix pylint warnings --- lib/tfw/builtins/frontend_event_handler.py | 3 ++- lib/tfw/builtins/ide_event_handler.py | 2 +- lib/tfw/builtins/log_monitoring_event_handler.py | 5 ++--- lib/tfw/components/file_manager/test_file_manager.py | 2 +- lib/tfw/components/history_monitor.py | 2 -- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/tfw/builtins/frontend_event_handler.py b/lib/tfw/builtins/frontend_event_handler.py index d3fe91e..1a76abe 100644 --- a/lib/tfw/builtins/frontend_event_handler.py +++ b/lib/tfw/builtins/frontend_event_handler.py @@ -1,9 +1,10 @@ from abc import ABC, abstractmethod from contextlib import suppress -from tfw.builtins.message_sender import MessageSender from tfw.event_handlers import FrontendEventHandlerBase +from .message_sender import MessageSender + class FrontendEventHandler(FrontendEventHandlerBase): def __init__(self): diff --git a/lib/tfw/builtins/ide_event_handler.py b/lib/tfw/builtins/ide_event_handler.py index f4025b0..e397874 100644 --- a/lib/tfw/builtins/ide_event_handler.py +++ b/lib/tfw/builtins/ide_event_handler.py @@ -4,7 +4,7 @@ import logging from tfw.networking import Scope -from tfw.event_handlers import FrontendEventHandlerBase, TFWServerUplinkConnector +from tfw.event_handlers import FrontendEventHandlerBase from tfw.components import FileManager from tfw.components.inotify import InotifyObserver diff --git a/lib/tfw/builtins/log_monitoring_event_handler.py b/lib/tfw/builtins/log_monitoring_event_handler.py index f1d2919..d2ce529 100644 --- a/lib/tfw/builtins/log_monitoring_event_handler.py +++ b/lib/tfw/builtins/log_monitoring_event_handler.py @@ -3,9 +3,8 @@ import logging -from os.path import dirname from tfw.networking import Scope -from tfw.event_handlers import FrontendEventHandlerBase, TFWServerUplinkConnector +from tfw.event_handlers import FrontendEventHandlerBase from tfw.components.inotify import InotifyObserver from tfw.mixins.supervisor_mixin import SupervisorLogMixin @@ -24,7 +23,7 @@ class LogInotifyObserver(InotifyObserver, SupervisorLogMixin): @property def process_name(self): return self._process_name - + @process_name.setter def process_name(self, process_name): self._process_name = process_name diff --git a/lib/tfw/components/file_manager/test_file_manager.py b/lib/tfw/components/file_manager/test_file_manager.py index 33c8024..380da0a 100644 --- a/lib/tfw/components/file_manager/test_file_manager.py +++ b/lib/tfw/components/file_manager/test_file_manager.py @@ -9,7 +9,7 @@ from tempfile import TemporaryDirectory import pytest -from file_manager import FileManager +from .file_manager import FileManager @dataclass class ManagerContext: diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index 703a68f..d8c5587 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -5,13 +5,11 @@ from pwd import getpwnam from grp import getgrnam from pathlib import Path from os import chown -from os.path import dirname from re import findall from re import compile as compileregex from abc import ABC, abstractmethod from tfw.components.inotify import InotifyObserver -from tfw.event_handlers import TFWServerUplinkConnector class HistoryMonitor(ABC, InotifyObserver): From 5f5fc1a8c9e9f9834ccedec5eaa541f2fd041093 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 27 Jun 2019 14:48:27 +0200 Subject: [PATCH 048/174] Refactor IdeEventHandler --- lib/tfw/builtins/ide_event_handler.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/tfw/builtins/ide_event_handler.py b/lib/tfw/builtins/ide_event_handler.py index e397874..f83e27c 100644 --- a/lib/tfw/builtins/ide_event_handler.py +++ b/lib/tfw/builtins/ide_event_handler.py @@ -10,7 +10,7 @@ from tfw.components.inotify import InotifyObserver LOG = logging.getLogger(__name__) -BUILD_ARTIFACTS = [ +BUILD_ARTIFACTS = ( "*.a", "*.class", "*.dll", @@ -31,7 +31,7 @@ BUILD_ARTIFACTS = [ "*.tar.gz", "*.zip", "*__pycache__*" -] +) class IdeEventHandler(FrontendEventHandlerBase): @@ -71,13 +71,7 @@ class IdeEventHandler(FrontendEventHandlerBase): ) self.monitor = InotifyObserver(self.filemanager.allowed_directories, exclude=BUILD_ARTIFACTS) - def on_modified(event): - LOG.debug(event) - self.server_connector.send_message({ - 'key': 'ide', - 'data': {'command': 'reload'} - }, Scope.WEBSOCKET) - self.monitor.on_modified = on_modified + self.monitor.on_modified = self._reload_frontend self.monitor.start() self.commands = { @@ -88,6 +82,12 @@ class IdeEventHandler(FrontendEventHandlerBase): 'exclude': self.exclude } + def _reload_frontend(self, event): + self.server_connector.send_message({ + 'key': 'ide', + 'data': {'command': 'reload'} + }, Scope.WEBSOCKET) + def read(self, data): """ Read the currently selected file. From c5c5d2c10ac46ea3ab75270d0a36cfa0eb4d7b5a Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 27 Jun 2019 15:44:23 +0200 Subject: [PATCH 049/174] Fix logging bug --- lib/tfw/config/log.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/tfw/config/log.py b/lib/tfw/config/log.py index c722cf4..119f7b8 100644 --- a/lib/tfw/config/log.py +++ b/lib/tfw/config/log.py @@ -109,10 +109,10 @@ class TFWLogFormatter(Formatter): def trim(self, value): if isinstance(value, dict): return {k: self.trim(v) for k, v in value.items()} - if isinstance(value, (int, float)): - return value - value_str = str(value) - return value_str if len(value_str) <= self.limit else f'{value_str[:self.limit]}...' + if isinstance(value, str): + value_str = str(value) + return value_str if len(value_str) <= self.limit else f'{value_str[:self.limit]}...' + return value @staticmethod def fetch_exception_origin(trace): From 6cc3d32097c3ccca6cff40494c6fac84483410fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 27 Jun 2019 15:22:48 +0200 Subject: [PATCH 050/174] Refactor LogMonitoringEventHandler --- .../builtins/log_monitoring_event_handler.py | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/lib/tfw/builtins/log_monitoring_event_handler.py b/lib/tfw/builtins/log_monitoring_event_handler.py index d2ce529..08f916c 100644 --- a/lib/tfw/builtins/log_monitoring_event_handler.py +++ b/lib/tfw/builtins/log_monitoring_event_handler.py @@ -13,12 +13,21 @@ LOG = logging.getLogger(__name__) class LogInotifyObserver(InotifyObserver, SupervisorLogMixin): def __init__(self, server_connector, process_name, log_tail=0): - self.prevent_log_recursion() - self.server_connector = server_connector + self._prevent_log_recursion() + self._server_connector = server_connector self._process_name = process_name self.log_tail = log_tail - self.procinfo = self.supervisor.getProcessInfo(self.process_name) - InotifyObserver.__init__(self, [self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']]) + self._procinfo = None + InotifyObserver.__init__(self, self._get_logfiles()) + + @staticmethod + def _prevent_log_recursion(): + # This is done to prevent inotify event logs triggering themselves (infinite log recursion) + logging.getLogger('watchdog.observers.inotify_buffer').propagate = False + + def _get_logfiles(self): + self._procinfo = self.supervisor.getProcessInfo(self._process_name) + return self._procinfo['stdout_logfile'], self._procinfo['stderr_logfile'] @property def process_name(self): @@ -27,11 +36,10 @@ class LogInotifyObserver(InotifyObserver, SupervisorLogMixin): @process_name.setter def process_name(self, process_name): self._process_name = process_name - self.procinfo = self.supervisor.getProcessInfo(process_name) - self.paths = [self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']] + self.paths = self._get_logfiles() def on_modified(self, event): - self.server_connector.send_message({ + self._server_connector.send_message({ 'key': 'processlog', 'data': { 'command': 'new_log', @@ -40,11 +48,6 @@ class LogInotifyObserver(InotifyObserver, SupervisorLogMixin): } }, Scope.BROADCAST) - @staticmethod - def prevent_log_recursion(): - # This is done to prevent inotify event logs triggering themselves (infinite log recursion) - logging.getLogger('watchdog.observers.inotify_buffer').propagate = False - class LogMonitoringEventHandler(FrontendEventHandlerBase): """ @@ -59,9 +62,8 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase): def __init__(self, key, process_name, log_tail=0): super().__init__(key) self.process_name = process_name - self.log_tail = log_tail - self.monitor = LogInotifyObserver(self.server_connector, process_name, log_tail) - self.monitor.start() + self._monitor = LogInotifyObserver(self.server_connector, process_name, log_tail) + self._monitor.start() self.command_handlers = { 'process_name': self.handle_process_name, @@ -82,7 +84,7 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase): :param data: TFW message data containing 'value' (name of the process to monitor) """ - self.monitor.process_name = data['value'] + self._monitor.process_name = data['value'] def handle_log_tail(self, data): """ @@ -93,7 +95,7 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase): :param data: TFW message data containing 'value' (new tail length) """ - self.monitor.log_tail = data['value'] + self._monitor.log_tail = data['value'] def cleanup(self): - self.monitor.stop() + self._monitor.stop() From da1976936208c0a0528031cb989777a65b85e0a2 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 27 Jun 2019 16:15:22 +0200 Subject: [PATCH 051/174] Fix file observer --- lib/tfw/components/history_monitor.py | 9 --------- lib/tfw/components/inotify/inotify.py | 12 ++++++------ 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index d8c5587..9855f57 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -1,10 +1,6 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from pwd import getpwnam -from grp import getgrnam -from pathlib import Path -from os import chown from re import findall from re import compile as compileregex from abc import ABC, abstractmethod @@ -30,11 +26,6 @@ class HistoryMonitor(ABC, InotifyObserver): self.history = [] self._last_length = len(self.history) self.uplink = uplink - uid = getpwnam('user').pw_uid - gid = getgrnam('users').gr_gid - path = Path(self.histfile) - path.touch() - chown(self.histfile, uid, gid) super().__init__(self.histfile) @property diff --git a/lib/tfw/components/inotify/inotify.py b/lib/tfw/components/inotify/inotify.py index 3bbefee..4e2d1dd 100644 --- a/lib/tfw/components/inotify/inotify.py +++ b/lib/tfw/components/inotify/inotify.py @@ -2,7 +2,7 @@ from typing import Iterable from time import time -from os.path import abspath, dirname, isfile +from os.path import abspath, dirname, isdir from watchdog.observers import Observer from watchdog.events import FileSystemMovedEvent, PatternMatchingEventHandler @@ -66,10 +66,10 @@ class InotifyDirDeletedEvent(InotifyEvent): class InotifyObserver: - def __init__(self, path, patterns=[], exclude=None, recursive=False): + def __init__(self, path, patterns=None, exclude=None, recursive=False): self._files = [] self._paths = path - self._patterns = patterns + self._patterns = patterns or [] self._exclude = exclude self._recursive = recursive self._observer = Observer() @@ -97,12 +97,12 @@ class InotifyObserver: def _extract_files_from_paths(self): files, paths = [], [] for path in self._paths: - if isfile(path): + if isdir(path): + paths.append(path) + else: new_file = abspath(path) files.append(new_file) paths.append(dirname(new_file)) - else: - paths.append(path) self._files, self._paths = files, paths @property From 44a1433d3b2a60acd7af6986b160d9293b399db1 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 27 Jun 2019 17:38:53 +0200 Subject: [PATCH 052/174] Refactor Inotify --- lib/tfw/components/inotify/inotify.py | 8 ++++---- lib/tfw/components/inotify/test_inotify.py | 10 +++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/tfw/components/inotify/inotify.py b/lib/tfw/components/inotify/inotify.py index 4e2d1dd..a989f83 100644 --- a/lib/tfw/components/inotify/inotify.py +++ b/lib/tfw/components/inotify/inotify.py @@ -97,12 +97,12 @@ class InotifyObserver: def _extract_files_from_paths(self): files, paths = [], [] for path in self._paths: + path = abspath(path) if isdir(path): paths.append(path) else: - new_file = abspath(path) - files.append(new_file) - paths.append(dirname(new_file)) + paths.append(dirname(path)) + files.append(path) self._files, self._paths = files, paths @property @@ -120,7 +120,7 @@ class InotifyObserver: @patterns.setter def patterns(self, patterns): - self._patterns = patterns + self._patterns = patterns or [] self._reset() @property diff --git a/lib/tfw/components/inotify/test_inotify.py b/lib/tfw/components/inotify/test_inotify.py index c6d0e12..aa910fe 100644 --- a/lib/tfw/components/inotify/test_inotify.py +++ b/lib/tfw/components/inotify/test_inotify.py @@ -5,7 +5,7 @@ from secrets import token_urlsafe from pathlib import Path from shutil import rmtree from os.path import join -from os import mkdir, rename +from os import mkdir, remove, rename from tempfile import TemporaryDirectory import watchdog @@ -140,11 +140,15 @@ def test_delete(context): def test_paths(context): context.observer.paths = context.subdir - context.create_random_folder(context.workdir) + newdir = context.create_random_folder(context.workdir) newfile = context.create_random_file(context.subdir, '.txt') + context.check_event(InotifyDirModifiedEvent, context.subdir) context.check_event(InotifyFileCreatedEvent, newfile) - context.observer.paths = context.subdir + context.observer.paths = [newdir, newfile] + remove(newfile) + context.check_event(InotifyFileDeletedEvent, newfile) assert context.check_any() + context.observer.paths = context.workdir def test_patterns(context): context.observer.patterns = ['*.txt'] From 491eaf6d5e46aff3b9b5639137634a01ca70d2dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 27 Jun 2019 16:19:51 +0200 Subject: [PATCH 053/174] Refactor HistoryMonitor --- lib/tfw/components/history_monitor.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index 9855f57..99add98 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -21,7 +21,6 @@ class HistoryMonitor(ABC, InotifyObserver): See examples below. """ def __init__(self, uplink, histfile): - self._domain = '' self.histfile = histfile self.history = [] self._last_length = len(self.history) @@ -29,12 +28,9 @@ class HistoryMonitor(ABC, InotifyObserver): super().__init__(self.histfile) @property + @abstractmethod def domain(self): - return self._domain - - @domain.setter - def domain(self, domain): - self._domain = domain + raise NotImplementedError() def on_modified(self, event): self._fetch_history() @@ -78,9 +74,9 @@ class BashMonitor(HistoryMonitor): shopt -s histappend unset HISTCONTROL """ - def __init__(self, uplink, histfile): - super().__init__(uplink, histfile) - self.domain = 'bash' + @property + def domain(self): + return 'bash' @property def command_pattern(self): @@ -95,9 +91,9 @@ class GDBMonitor(HistoryMonitor): HistoryMonitor to monitor GDB sessions. For this to work "set trace-commands on" must be set in GDB. """ - def __init__(self, histfile): - super().__init__(histfile) - self.domain = 'gdb' + @property + def domain(self): + return 'gdb' @property def command_pattern(self): From bcee48646346e450213e725fff7fa5dd8e3969f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 27 Jun 2019 16:36:39 +0200 Subject: [PATCH 054/174] Refactor InotifyObserver --- lib/tfw/components/inotify/inotify.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/tfw/components/inotify/inotify.py b/lib/tfw/components/inotify/inotify.py index a989f83..4f69a02 100644 --- a/lib/tfw/components/inotify/inotify.py +++ b/lib/tfw/components/inotify/inotify.py @@ -76,11 +76,6 @@ class InotifyObserver: self._reset() def _reset(self): - dispatch_event = self._dispatch_event - class TransformerEventHandler(PatternMatchingEventHandler): - def on_any_event(self, event): - dispatch_event(event) - if isinstance(self._paths, str): self._paths = [self._paths] if isinstance(self._paths, Iterable): @@ -89,10 +84,11 @@ class InotifyObserver: raise ValueError('Expected one or more string paths.') patterns = self._files+self.patterns - self.handler = TransformerEventHandler(patterns if patterns else None, self.exclude) + handler = PatternMatchingEventHandler(patterns if patterns else None, self.exclude) + handler.on_any_event = self._dispatch_event self._observer.unschedule_all() for path in self.paths: - self._observer.schedule(self.handler, path, self._recursive) + self._observer.schedule(handler, path, self._recursive) def _extract_files_from_paths(self): files, paths = [], [] From 28c3e68b5a3eb8f22229154e26b01077f53ae961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 27 Jun 2019 17:18:04 +0200 Subject: [PATCH 055/174] Refactor TFWLog --- lib/tfw/config/log.py | 52 ++++++++++--------------------------------- 1 file changed, 12 insertions(+), 40 deletions(-) diff --git a/lib/tfw/config/log.py b/lib/tfw/config/log.py index 119f7b8..1c01a7f 100644 --- a/lib/tfw/config/log.py +++ b/lib/tfw/config/log.py @@ -1,29 +1,18 @@ # pylint: disable=bad-whitespace from sys import stderr -from collections import deque from datetime import datetime -from traceback import format_exception, walk_tb from logging import DEBUG, getLogger, Handler, Formatter, Filter from .envvars import TFWENV -class COLOR: - BLACK = '\033[30m' +class Color: GREY = '\033[30;1m' RED = '\033[31m' BOLDRED = '\033[31;1m' - GREEN = '\033[32m' BOLDGREEN = '\033[32;1m' - ORANGE = '\033[33m' YELLOW = '\033[33;1m' - BLUE = '\033[34m' - BOLDBLUE = '\033[34;1m' - MAGENTA = '\033[35m' - BOLDMAGENTA = '\033[35;1m' CYAN = '\033[36m' - BOLDCYAN = '\033[36;1m' - WHITE = '\033[37m' BOLDWHITE = '\033[37;1m' RESET = '\033[0m' @@ -63,17 +52,16 @@ class TFWLogHandler(Handler): class TFWLogFormatter(Formatter): severity_to_color = { - 'CRITICAL' : COLOR.BOLDRED, - 'ERROR' : COLOR.RED, - 'WARNING' : COLOR.YELLOW, - 'INFO' : COLOR.BOLDGREEN, - 'DEBUG' : COLOR.BOLDWHITE, - 'NOTSET' : COLOR.CYAN + 'CRITICAL' : Color.BOLDRED, + 'ERROR' : Color.RED, + 'WARNING' : Color.YELLOW, + 'INFO' : Color.BOLDGREEN, + 'DEBUG' : Color.BOLDWHITE, + 'NOTSET' : Color.CYAN } def __init__(self, limit): self.limit = limit - self.last_trace = None super().__init__() def format(self, record): @@ -87,23 +75,11 @@ class TFWLogFormatter(Formatter): short_message = record.msg long_message = record.msg - if record.exc_info: - current_trace = self.fetch_exception_origin(record.exc_info[2]) - if current_trace != self.last_trace: - self.last_trace = current_trace - trace = '\n'+''.join(format_exception(*record.exc_info)) - else: - trace = (f'\nSee previous traceback...\n' - f'{record.exc_info[0].__name__}: {record.exc_info[1]}') - else: - trace = '' - - short_entry = (f'[{COLOR.GREY}{date}{COLOR.RESET}|>' + short_entry = (f'[{Color.GREY}{date}{Color.RESET}|>' f'{self.severity_to_color[record.levelname]}{record.module}:' - f'{record.levelname.lower()}{COLOR.RESET}] {short_message}' - f'{trace}') + f'{record.levelname.lower()}{Color.RESET}] {short_message}') long_entry = (f'[{date}|>{record.module}:{record.levelname.lower()}] ' - f'{long_message}{trace}') + f'{long_message}') return short_entry, long_entry def trim(self, value): @@ -114,12 +90,8 @@ class TFWLogFormatter(Formatter): return value_str if len(value_str) <= self.limit else f'{value_str[:self.limit]}...' return value - @staticmethod - def fetch_exception_origin(trace): - return deque(walk_tb(trace), maxlen=1).pop() - -class TFWLogWhitelistFilter(Filter): +class WhitelistFilter(Filter): def __init__(self, names): self.names = names super().__init__() @@ -128,7 +100,7 @@ class TFWLogWhitelistFilter(Filter): return record.module in self.names -class TFWLogBlacklistFilter(Filter): +class BlacklistFilter(Filter): def __init__(self, names): self.names = names super().__init__() From 78bd97493ab45ce9c5213746baae29545ffe43ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 28 Jun 2019 15:11:02 +0200 Subject: [PATCH 056/174] Rework EventHandler situation --- .pylintrc | 2 +- lib/tfw/builtins/__init__.py | 7 ++-- .../directory_snapshotting_event_handler.py | 8 +++-- lib/tfw/builtins/event_handler.py | 10 ++++++ lib/tfw/builtins/frontend_event_handler.py | 7 ++-- .../fsm_aware_event_handler.py | 12 ++++--- .../builtins/fsm_managing_event_handler.py | 7 ++-- lib/tfw/builtins/ide_event_handler.py | 19 +++++++---- .../builtins/log_monitoring_event_handler.py | 7 ++-- lib/tfw/builtins/message_sender.py | 2 +- lib/tfw/builtins/pipe_io_event_handler.py | 5 +-- .../process_managing_event_handler.py | 8 +++-- lib/tfw/builtins/terminal_event_handler.py | 8 +++-- .../tfw_server_connector.py | 0 lib/tfw/components/__init__.py | 1 + .../fsm_aware.py | 0 lib/tfw/event_handlers/__init__.py | 4 --- .../boradcasting_event_handler.py | 32 ------------------- lib/tfw/event_handlers/event_handler_base.py | 13 ++++---- .../frontend_event_handler_base.py | 8 ----- 20 files changed, 74 insertions(+), 86 deletions(-) create mode 100644 lib/tfw/builtins/event_handler.py rename lib/tfw/{event_handlers => builtins}/fsm_aware_event_handler.py (66%) rename lib/tfw/{event_handlers => builtins}/tfw_server_connector.py (100%) rename lib/tfw/{event_handlers => components}/fsm_aware.py (100%) delete mode 100644 lib/tfw/event_handlers/boradcasting_event_handler.py delete mode 100644 lib/tfw/event_handlers/frontend_event_handler_base.py diff --git a/.pylintrc b/.pylintrc index 1cbae5d..961a526 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,7 +6,7 @@ disable = missing-docstring, too-few-public-methods, invalid-name [SIMILARITIES] -min-similarity-lines=6 +min-similarity-lines=7 ignore-comments=yes ignore-docstrings=yes ignore-imports=yes diff --git a/lib/tfw/builtins/__init__.py b/lib/tfw/builtins/__init__.py index c554666..b4e7178 100644 --- a/lib/tfw/builtins/__init__.py +++ b/lib/tfw/builtins/__init__.py @@ -1,10 +1,13 @@ +from .tfw_server_connector import TFWServerUplinkConnector, TFWServerConnector +from .event_handler import EventHandler +from .fsm_aware_event_handler import FSMAwareEventHandler from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler from .frontend_event_handler import FrontendEventHandler from .fsm_managing_event_handler import FSMManagingEventHandler from .ide_event_handler import IdeEventHandler from .log_monitoring_event_handler import LogMonitoringEventHandler from .message_sender import MessageSender -from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler -from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler +#from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler +#from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler from .process_managing_event_handler import ProcessManagingEventHandler from .terminal_event_handler import TerminalEventHandler diff --git a/lib/tfw/builtins/directory_snapshotting_event_handler.py b/lib/tfw/builtins/directory_snapshotting_event_handler.py index 1e8c1ea..80b6b34 100644 --- a/lib/tfw/builtins/directory_snapshotting_event_handler.py +++ b/lib/tfw/builtins/directory_snapshotting_event_handler.py @@ -9,16 +9,18 @@ from datetime import datetime from dateutil import parser as dateparser -from tfw.event_handlers import FrontendEventHandlerBase from tfw.components.snapshot_provider import SnapshotProvider from tfw.config import TFWENV +from tfw.networking import Scope + +from .event_handler import EventHandler LOG = logging.getLogger(__name__) -class DirectorySnapshottingEventHandler(FrontendEventHandlerBase): +class DirectorySnapshottingEventHandler(EventHandler): def __init__(self, key, directories, exclude_unix_patterns=None): - super().__init__(key) + super().__init__(key, scope=Scope.WEBSOCKET) self.snapshot_providers = {} self._exclude_unix_patterns = exclude_unix_patterns self.init_snapshot_providers(directories) diff --git a/lib/tfw/builtins/event_handler.py b/lib/tfw/builtins/event_handler.py new file mode 100644 index 0000000..1732044 --- /dev/null +++ b/lib/tfw/builtins/event_handler.py @@ -0,0 +1,10 @@ +from tfw.event_handlers import EventHandlerBase +from tfw.networking import Scope + +from .tfw_server_connector import TFWServerConnector + + +class EventHandler(EventHandlerBase): + # pylint: disable=abstract-method + def __init__(self, key, scope=Scope.ZMQ): + super().__init__(key, TFWServerConnector(), scope=scope) diff --git a/lib/tfw/builtins/frontend_event_handler.py b/lib/tfw/builtins/frontend_event_handler.py index 1a76abe..9e46869 100644 --- a/lib/tfw/builtins/frontend_event_handler.py +++ b/lib/tfw/builtins/frontend_event_handler.py @@ -1,16 +1,17 @@ from abc import ABC, abstractmethod from contextlib import suppress -from tfw.event_handlers import FrontendEventHandlerBase +from tfw.networking import Scope from .message_sender import MessageSender +from .event_handler import EventHandler -class FrontendEventHandler(FrontendEventHandlerBase): +class FrontendEventHandler(EventHandler): def __init__(self): frontend_keys = ('message', 'queueMessages', 'dashboard', 'console') self._frontend_message_storage = FrontendMessageStorage(frontend_keys) - super().__init__((*frontend_keys, 'recover')) + super().__init__((*frontend_keys, 'recover'), scope=Scope.WEBSOCKET) def handle_event(self, message): self._frontend_message_storage.save_message(message) diff --git a/lib/tfw/event_handlers/fsm_aware_event_handler.py b/lib/tfw/builtins/fsm_aware_event_handler.py similarity index 66% rename from lib/tfw/event_handlers/fsm_aware_event_handler.py rename to lib/tfw/builtins/fsm_aware_event_handler.py index 60313df..01d8410 100644 --- a/lib/tfw/event_handlers/fsm_aware_event_handler.py +++ b/lib/tfw/builtins/fsm_aware_event_handler.py @@ -3,18 +3,20 @@ from abc import ABC -from .event_handler_base import EventHandlerBase -from .fsm_aware import FSMAware +from tfw.components import FSMAware +from tfw.networking import Scope + +from .event_handler import EventHandler -class FSMAwareEventHandler(EventHandlerBase, FSMAware, ABC): +class FSMAwareEventHandler(EventHandler, FSMAware, ABC): # pylint: disable=abstract-method """ Abstract base class for EventHandlers which automatically keep track of the state of the TFW FSM. """ - def __init__(self, key): - EventHandlerBase.__init__(self, key) + def __init__(self, key, scope=Scope.ZMQ): + EventHandler.__init__(self, key, scope=scope) FSMAware.__init__(self) self.subscribe('fsm_update') diff --git a/lib/tfw/builtins/fsm_managing_event_handler.py b/lib/tfw/builtins/fsm_managing_event_handler.py index 4860f5a..a4814f9 100644 --- a/lib/tfw/builtins/fsm_managing_event_handler.py +++ b/lib/tfw/builtins/fsm_managing_event_handler.py @@ -3,14 +3,15 @@ import logging -from tfw.event_handlers import FrontendEventHandlerBase from tfw.crypto import KeyManager, sign_message, verify_message from tfw.networking import Scope +from .event_handler import EventHandler + LOG = logging.getLogger(__name__) -class FSMManagingEventHandler(FrontendEventHandlerBase): +class FSMManagingEventHandler(EventHandler): """ EventHandler responsible for managing the state machine of the framework (TFW FSM). @@ -26,7 +27,7 @@ class FSMManagingEventHandler(FrontendEventHandlerBase): command. """ def __init__(self, key, fsm_type, require_signature=False): - super().__init__(key) + super().__init__(key, scope=Scope.WEBSOCKET) self.fsm = fsm_type() self._fsm_updater = FSMUpdater(self.fsm) self.auth_key = KeyManager().auth_key diff --git a/lib/tfw/builtins/ide_event_handler.py b/lib/tfw/builtins/ide_event_handler.py index f83e27c..dd257d3 100644 --- a/lib/tfw/builtins/ide_event_handler.py +++ b/lib/tfw/builtins/ide_event_handler.py @@ -4,10 +4,11 @@ import logging from tfw.networking import Scope -from tfw.event_handlers import FrontendEventHandlerBase from tfw.components import FileManager from tfw.components.inotify import InotifyObserver +from .event_handler import EventHandler + LOG = logging.getLogger(__name__) BUILD_ARTIFACTS = ( @@ -34,7 +35,7 @@ BUILD_ARTIFACTS = ( ) -class IdeEventHandler(FrontendEventHandlerBase): +class IdeEventHandler(EventHandler): # pylint: disable=too-many-arguments,anomalous-backslash-in-string """ Event handler implementing the backend of our browser based IDE. @@ -57,7 +58,7 @@ class IdeEventHandler(FrontendEventHandlerBase): :param selected_file: file that is selected by default :param exclude: list of filenames that should not appear between files (for .o, .pyc, etc.) """ - super().__init__(key) + super().__init__(key, scope=Scope.WEBSOCKET) try: self.filemanager = FileManager( allowed_directories=allowed_directories, @@ -70,7 +71,10 @@ class IdeEventHandler(FrontendEventHandlerBase): f'No file(s) in IdeEventHandler working_directory "{directory}"!' ) - self.monitor = InotifyObserver(self.filemanager.allowed_directories, exclude=BUILD_ARTIFACTS) + self.monitor = InotifyObserver( + self.filemanager.allowed_directories, + exclude=BUILD_ARTIFACTS + ) self.monitor.on_modified = self._reload_frontend self.monitor.start() @@ -82,7 +86,7 @@ class IdeEventHandler(FrontendEventHandlerBase): 'exclude': self.exclude } - def _reload_frontend(self, event): + def _reload_frontend(self, event): # pylint: disable=unused-argument self.server_connector.send_message({ 'key': 'ide', 'data': {'command': 'reload'} @@ -150,7 +154,10 @@ class IdeEventHandler(FrontendEventHandlerBase): except IndexError: data['content'] = 'No files in this directory :(' except EnvironmentError as err: - LOG.error('Failed to select directory "%s". Reason: %s', data['directory'], str(err)) + LOG.error( + 'Failed to select directory "%s". Reason: %s', + data['directory'], str(err) + ) return data def exclude(self, data): diff --git a/lib/tfw/builtins/log_monitoring_event_handler.py b/lib/tfw/builtins/log_monitoring_event_handler.py index 08f916c..85699c8 100644 --- a/lib/tfw/builtins/log_monitoring_event_handler.py +++ b/lib/tfw/builtins/log_monitoring_event_handler.py @@ -4,10 +4,11 @@ import logging from tfw.networking import Scope -from tfw.event_handlers import FrontendEventHandlerBase from tfw.components.inotify import InotifyObserver from tfw.mixins.supervisor_mixin import SupervisorLogMixin +from .event_handler import EventHandler + LOG = logging.getLogger(__name__) @@ -49,7 +50,7 @@ class LogInotifyObserver(InotifyObserver, SupervisorLogMixin): }, Scope.BROADCAST) -class LogMonitoringEventHandler(FrontendEventHandlerBase): +class LogMonitoringEventHandler(EventHandler): """ Monitors the output of a supervisor process (stdout, stderr) and sends the results to the frontend. @@ -60,7 +61,7 @@ class LogMonitoringEventHandler(FrontendEventHandlerBase): The API of each command is documented in their respective handler. """ def __init__(self, key, process_name, log_tail=0): - super().__init__(key) + super().__init__(key, scope=Scope.WEBSOCKET) self.process_name = process_name self._monitor = LogInotifyObserver(self.server_connector, process_name, log_tail) self._monitor.start() diff --git a/lib/tfw/builtins/message_sender.py b/lib/tfw/builtins/message_sender.py index 4f35baa..a4ee179 100644 --- a/lib/tfw/builtins/message_sender.py +++ b/lib/tfw/builtins/message_sender.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw.event_handlers import TFWServerUplinkConnector +from .tfw_server_connector import TFWServerUplinkConnector class MessageSender: diff --git a/lib/tfw/builtins/pipe_io_event_handler.py b/lib/tfw/builtins/pipe_io_event_handler.py index 9ab4db5..96bff90 100644 --- a/lib/tfw/builtins/pipe_io_event_handler.py +++ b/lib/tfw/builtins/pipe_io_event_handler.py @@ -10,14 +10,15 @@ from secrets import token_urlsafe from threading import Thread from contextlib import suppress -from tfw.event_handlers import EventHandlerBase from tfw.components.pipe_io_server import PipeIOServer, terminate_process_on_failure +from .event_handler import EventHandler + LOG = logging.getLogger(__name__) DEFAULT_PERMISSIONS = 0o600 -class PipeIOEventHandlerBase(EventHandlerBase): +class PipeIOEventHandlerBase(EventHandler): def __init__(self, key, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS): super().__init__(key) self.pipe_io = CallbackPipeIOServer( diff --git a/lib/tfw/builtins/process_managing_event_handler.py b/lib/tfw/builtins/process_managing_event_handler.py index 4c7daaa..0109e7a 100644 --- a/lib/tfw/builtins/process_managing_event_handler.py +++ b/lib/tfw/builtins/process_managing_event_handler.py @@ -4,9 +4,11 @@ import logging from xmlrpc.client import Fault as SupervisorFault -from tfw.event_handlers import FrontendEventHandlerBase +from tfw.networking import Scope from tfw.mixins.supervisor_mixin import SupervisorMixin, SupervisorLogMixin +from .event_handler import EventHandler + LOG = logging.getLogger(__name__) @@ -22,7 +24,7 @@ class ProcessManager(SupervisorMixin, SupervisorLogMixin): return self.commands[command](process_name) -class ProcessManagingEventHandler(FrontendEventHandlerBase): +class ProcessManagingEventHandler(EventHandler): """ Event handler that can manage processes managed by supervisor. @@ -36,7 +38,7 @@ class ProcessManagingEventHandler(FrontendEventHandlerBase): (the names are as self-documenting as it gets) """ def __init__(self, key, log_tail=0): - super().__init__(key) + super().__init__(key, scope=Scope.WEBSOCKET) self.processmanager = ProcessManager() self.log_tail = log_tail diff --git a/lib/tfw/builtins/terminal_event_handler.py b/lib/tfw/builtins/terminal_event_handler.py index 6d2a14a..68a6b78 100644 --- a/lib/tfw/builtins/terminal_event_handler.py +++ b/lib/tfw/builtins/terminal_event_handler.py @@ -3,16 +3,18 @@ import logging -from tfw.event_handlers import FrontendEventHandlerBase +from tfw.networking import Scope from tfw.components import BashMonitor from tfw.components.terminado_mini_server import TerminadoMiniServer from tfw.config import TFWENV from tao.config import TAOENV +from .event_handler import EventHandler + LOG = logging.getLogger(__name__) -class TerminalEventHandler(FrontendEventHandlerBase): +class TerminalEventHandler(EventHandler): """ Event handler responsible for managing terminal sessions for frontend xterm sessions to connect to. You need to instanciate this in order for frontend @@ -27,7 +29,7 @@ class TerminalEventHandler(FrontendEventHandlerBase): :param key: key this EventHandler listens to :param monitor: tfw.components.HistoryMonitor instance to read command history from """ - super().__init__(key) + super().__init__(key, scope=Scope.WEBSOCKET) self._historymonitor = BashMonitor(self.server_connector, TFWENV.HISTFILE) bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash'] diff --git a/lib/tfw/event_handlers/tfw_server_connector.py b/lib/tfw/builtins/tfw_server_connector.py similarity index 100% rename from lib/tfw/event_handlers/tfw_server_connector.py rename to lib/tfw/builtins/tfw_server_connector.py diff --git a/lib/tfw/components/__init__.py b/lib/tfw/components/__init__.py index 203f7e1..75e6531 100644 --- a/lib/tfw/components/__init__.py +++ b/lib/tfw/components/__init__.py @@ -6,3 +6,4 @@ from .file_manager import FileManager from .history_monitor import HistoryMonitor, BashMonitor, GDBMonitor from .snapshot_provider import SnapshotProvider from .terminal_commands import TerminalCommands +from .fsm_aware import FSMAware diff --git a/lib/tfw/event_handlers/fsm_aware.py b/lib/tfw/components/fsm_aware.py similarity index 100% rename from lib/tfw/event_handlers/fsm_aware.py rename to lib/tfw/components/fsm_aware.py diff --git a/lib/tfw/event_handlers/__init__.py b/lib/tfw/event_handlers/__init__.py index 0327aef..447f617 100644 --- a/lib/tfw/event_handlers/__init__.py +++ b/lib/tfw/event_handlers/__init__.py @@ -2,7 +2,3 @@ # All Rights Reserved. See LICENSE file for details. from .event_handler_base import EventHandlerBase -from .frontend_event_handler_base import FrontendEventHandlerBase -from .boradcasting_event_handler import BroadcastingEventHandler -from .fsm_aware_event_handler import FSMAwareEventHandler -from .tfw_server_connector import TFWServerUplinkConnector, TFWServerConnector diff --git a/lib/tfw/event_handlers/boradcasting_event_handler.py b/lib/tfw/event_handlers/boradcasting_event_handler.py deleted file mode 100644 index c6f61a9..0000000 --- a/lib/tfw/event_handlers/boradcasting_event_handler.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -from abc import ABC - -from tfw.networking import Scope -from tfw.crypto import message_checksum - -from .event_handler_base import EventHandlerBase - - -class BroadcastingEventHandler(EventHandlerBase, ABC): - # pylint: disable=abstract-method - """ - Abstract base class for EventHandlers which broadcast responses - and intelligently ignore their own broadcasted messages they receive. - """ - def __init__(self, key): - super().__init__(key) - self.own_message_hashes = [] - - def event_handler_callback(self, message): - message_hash = message_checksum(message) - - if message_hash in self.own_message_hashes: - self.own_message_hashes.remove(message_hash) - return - - response = self.dispatch_handling(message) - if response: - self.own_message_hashes.append(message_checksum(response)) - self.server_connector.send_message(response, Scope.BROADCAST) diff --git a/lib/tfw/event_handlers/event_handler_base.py b/lib/tfw/event_handlers/event_handler_base.py index d5bb20e..cc0be96 100644 --- a/lib/tfw/event_handlers/event_handler_base.py +++ b/lib/tfw/event_handlers/event_handler_base.py @@ -2,24 +2,24 @@ # All Rights Reserved. See LICENSE file for details. import logging -from abc import ABC, abstractmethod from inspect import currentframe from typing import Iterable -from .tfw_server_connector import TFWServerConnector +from tfw.networking import Scope LOG = logging.getLogger(__name__) -class EventHandlerBase(ABC): +class EventHandlerBase: """ Abstract base class for all Python based EventHandlers. Useful implementation template for other languages. Derived classes must implement the handle_event() method """ - def __init__(self, key): - self.server_connector = TFWServerConnector() + def __init__(self, key, server_connector, scope=Scope.ZMQ): + self.server_connector = server_connector + self.scope = scope self.keys = [] if isinstance(key, str): self.keys.append(key) @@ -51,7 +51,7 @@ class EventHandlerBase(ABC): self.send_message(response) def send_message(self, message): - self.server_connector.send_message(message) + self.server_connector.send_message(message, self.scope) def check_key(self, message): """ @@ -76,7 +76,6 @@ class EventHandlerBase(ABC): """ return self.handle_event(message) - @abstractmethod def handle_event(self, message): """ Abstract method that implements the handling of messages. diff --git a/lib/tfw/event_handlers/frontend_event_handler_base.py b/lib/tfw/event_handlers/frontend_event_handler_base.py deleted file mode 100644 index 6990632..0000000 --- a/lib/tfw/event_handlers/frontend_event_handler_base.py +++ /dev/null @@ -1,8 +0,0 @@ -from tfw.networking import Scope - -from .event_handler_base import EventHandlerBase - - -class FrontendEventHandlerBase(EventHandlerBase): # pylint: disable=abstract-method - def send_message(self, message): - self.server_connector.send_message(message, Scope.WEBSOCKET) From 481a91960659fd74420daaf907844be9a5bccd25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 28 Jun 2019 15:16:08 +0200 Subject: [PATCH 057/174] Improve FSMAware api --- lib/tfw/builtins/__init__.py | 4 ++-- lib/tfw/builtins/fsm_aware_event_handler.py | 2 +- lib/tfw/components/fsm_aware.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/tfw/builtins/__init__.py b/lib/tfw/builtins/__init__.py index b4e7178..af322b8 100644 --- a/lib/tfw/builtins/__init__.py +++ b/lib/tfw/builtins/__init__.py @@ -7,7 +7,7 @@ from .fsm_managing_event_handler import FSMManagingEventHandler from .ide_event_handler import IdeEventHandler from .log_monitoring_event_handler import LogMonitoringEventHandler from .message_sender import MessageSender -#from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler -#from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler +from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler +from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler from .process_managing_event_handler import ProcessManagingEventHandler from .terminal_event_handler import TerminalEventHandler diff --git a/lib/tfw/builtins/fsm_aware_event_handler.py b/lib/tfw/builtins/fsm_aware_event_handler.py index 01d8410..6eb7565 100644 --- a/lib/tfw/builtins/fsm_aware_event_handler.py +++ b/lib/tfw/builtins/fsm_aware_event_handler.py @@ -21,6 +21,6 @@ class FSMAwareEventHandler(EventHandler, FSMAware, ABC): self.subscribe('fsm_update') def dispatch_handling(self, message): - if self.update_fsm_data(message): + if self.refresh_on_fsm_update(message): return None return super().dispatch_handling(message) diff --git a/lib/tfw/components/fsm_aware.py b/lib/tfw/components/fsm_aware.py index ddbf6fb..beb4957 100644 --- a/lib/tfw/components/fsm_aware.py +++ b/lib/tfw/components/fsm_aware.py @@ -19,7 +19,7 @@ class FSMAware: self.fsm_event_log = [] self._auth_key = KeyManager().auth_key - def update_fsm_data(self, message): + def refresh_on_fsm_update(self, message): if message['key'] == 'fsm_update' and verify_message(self._auth_key, message): self._handle_fsm_update(message) return True From 018286f3971764af937cbc118d92db820631bd0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 28 Jun 2019 16:50:36 +0200 Subject: [PATCH 058/174] Improve EventHandler response sending pattern --- .../directory_snapshotting_event_handler.py | 2 +- lib/tfw/builtins/frontend_event_handler.py | 2 +- lib/tfw/builtins/fsm_aware_event_handler.py | 5 +-- .../builtins/fsm_managing_event_handler.py | 2 +- lib/tfw/builtins/ide_event_handler.py | 2 +- .../process_managing_event_handler.py | 2 +- lib/tfw/builtins/terminal_event_handler.py | 2 +- lib/tfw/event_handlers/event_handler_base.py | 45 +++++++------------ 8 files changed, 24 insertions(+), 38 deletions(-) diff --git a/lib/tfw/builtins/directory_snapshotting_event_handler.py b/lib/tfw/builtins/directory_snapshotting_event_handler.py index 80b6b34..81683c5 100644 --- a/lib/tfw/builtins/directory_snapshotting_event_handler.py +++ b/lib/tfw/builtins/directory_snapshotting_event_handler.py @@ -53,7 +53,7 @@ class DirectorySnapshottingEventHandler(EventHandler): try: data = message['data'] message['data'] = self.command_handlers[data['command']](data) - return message + self.send_message(message) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/lib/tfw/builtins/frontend_event_handler.py b/lib/tfw/builtins/frontend_event_handler.py index 9e46869..1099564 100644 --- a/lib/tfw/builtins/frontend_event_handler.py +++ b/lib/tfw/builtins/frontend_event_handler.py @@ -17,7 +17,7 @@ class FrontendEventHandler(EventHandler): self._frontend_message_storage.save_message(message) if message['key'] == 'recover': self.recover_frontend() - return message + self.send_message(message) def recover_frontend(self): for message in self._frontend_message_storage.messages: diff --git a/lib/tfw/builtins/fsm_aware_event_handler.py b/lib/tfw/builtins/fsm_aware_event_handler.py index 6eb7565..ac467ed 100644 --- a/lib/tfw/builtins/fsm_aware_event_handler.py +++ b/lib/tfw/builtins/fsm_aware_event_handler.py @@ -21,6 +21,5 @@ class FSMAwareEventHandler(EventHandler, FSMAware, ABC): self.subscribe('fsm_update') def dispatch_handling(self, message): - if self.refresh_on_fsm_update(message): - return None - return super().dispatch_handling(message) + if not self.refresh_on_fsm_update(message): + super().dispatch_handling(message) diff --git a/lib/tfw/builtins/fsm_managing_event_handler.py b/lib/tfw/builtins/fsm_managing_event_handler.py index a4814f9..8b5f4d6 100644 --- a/lib/tfw/builtins/fsm_managing_event_handler.py +++ b/lib/tfw/builtins/fsm_managing_event_handler.py @@ -46,7 +46,7 @@ class FSMManagingEventHandler(EventHandler): sign_message(self.auth_key, message) sign_message(self.auth_key, fsm_update_message) self.server_connector.send_message(fsm_update_message, Scope.BROADCAST) - return message + self.send_message(message) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/lib/tfw/builtins/ide_event_handler.py b/lib/tfw/builtins/ide_event_handler.py index dd257d3..e03fc17 100644 --- a/lib/tfw/builtins/ide_event_handler.py +++ b/lib/tfw/builtins/ide_event_handler.py @@ -187,7 +187,7 @@ class IdeEventHandler(EventHandler): data = message['data'] message['data'] = self.commands[data['command']](data) self.attach_fileinfo(data) - return message + self.send_message(message) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/lib/tfw/builtins/process_managing_event_handler.py b/lib/tfw/builtins/process_managing_event_handler.py index 0109e7a..5f687cd 100644 --- a/lib/tfw/builtins/process_managing_event_handler.py +++ b/lib/tfw/builtins/process_managing_event_handler.py @@ -58,6 +58,6 @@ class ProcessManagingEventHandler(EventHandler): data['process_name'], self.log_tail ) - return message + self.send_message(message) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/lib/tfw/builtins/terminal_event_handler.py b/lib/tfw/builtins/terminal_event_handler.py index 68a6b78..9b0df08 100644 --- a/lib/tfw/builtins/terminal_event_handler.py +++ b/lib/tfw/builtins/terminal_event_handler.py @@ -56,7 +56,7 @@ class TerminalEventHandler(EventHandler): try: data = message['data'] message['data'] = self.commands[data['command']](data) - return message + self.send_message(message) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/lib/tfw/event_handlers/event_handler_base.py b/lib/tfw/event_handlers/event_handler_base.py index cc0be96..281c92f 100644 --- a/lib/tfw/event_handlers/event_handler_base.py +++ b/lib/tfw/event_handlers/event_handler_base.py @@ -5,8 +5,6 @@ import logging from inspect import currentframe from typing import Iterable -from tfw.networking import Scope - LOG = logging.getLogger(__name__) @@ -17,7 +15,7 @@ class EventHandlerBase: Derived classes must implement the handle_event() method """ - def __init__(self, key, server_connector, scope=Scope.ZMQ): + def __init__(self, key, server_connector, scope): self.server_connector = server_connector self.scope = scope self.keys = [] @@ -29,12 +27,18 @@ class EventHandlerBase: self.subscribe(*self.keys) self.server_connector.register_callback(self.event_handler_callback) - @property - def key(self): + def subscribe(self, *keys): """ - Returns the oldest key this EventHandler was subscribed to. + Subscribe this EventHandler to receive events for given keys. + Note that you can subscribe to the same key several times in which + case you will need to unsubscribe multiple times in order to stop + receiving events. + + :param keys: list of keys to subscribe to """ - return self.keys[0] + for key in keys: + self.server_connector.subscribe(key) + self.keys.append(key) def event_handler_callback(self, message): """ @@ -43,15 +47,8 @@ class EventHandlerBase: a response back in case the handler returned something. This is subscribed in __init__(). """ - if not self.check_key(message): - return - - response = self.dispatch_handling(message) - if response: - self.send_message(response) - - def send_message(self, message): - self.server_connector.send_message(message, self.scope) + if self.check_key(message): + self.dispatch_handling(message) def check_key(self, message): """ @@ -74,7 +71,7 @@ class EventHandlerBase: :param message: the message received :returns: the message to send back """ - return self.handle_event(message) + self.handle_event(message) def handle_event(self, message): """ @@ -85,18 +82,8 @@ class EventHandlerBase: """ raise NotImplementedError - def subscribe(self, *keys): - """ - Subscribe this EventHandler to receive events for given keys. - Note that you can subscribe to the same key several times in which - case you will need to unsubscribe multiple times in order to stop - receiving events. - - :param keys: list of keys to subscribe to - """ - for key in keys: - self.server_connector.subscribe(key) - self.keys.append(key) + def send_message(self, message): + self.server_connector.send_message(message, self.scope) def unsubscribe(self, *keys): """ From aafd145fc56d9454a368b8277550e093e18b2a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 28 Jun 2019 17:00:00 +0200 Subject: [PATCH 059/174] Delete copyright headers in preparation of going open source ^^ --- Dockerfile | 2 +- LICENSE | 12 ------------ lib/envvars/__init__.py | 3 --- lib/tao/__init__.py | 2 -- lib/tao/config/__init__.py | 3 --- lib/tao/config/envvars.py | 3 --- lib/tfw/__init__.py | 2 -- .../builtins/directory_snapshotting_event_handler.py | 3 --- lib/tfw/builtins/fsm_aware_event_handler.py | 3 --- lib/tfw/builtins/fsm_managing_event_handler.py | 3 --- lib/tfw/builtins/ide_event_handler.py | 3 --- lib/tfw/builtins/log_monitoring_event_handler.py | 3 --- lib/tfw/builtins/message_sender.py | 3 --- lib/tfw/builtins/process_managing_event_handler.py | 3 --- lib/tfw/builtins/terminal_event_handler.py | 3 --- lib/tfw/components/__init__.py | 3 --- lib/tfw/components/commands_equal.py | 3 --- lib/tfw/components/fsm_aware.py | 3 --- lib/tfw/components/history_monitor.py | 3 --- lib/tfw/components/snapshot_provider.py | 3 --- lib/tfw/components/terminado_mini_server.py | 3 --- lib/tfw/components/terminal_commands.py | 3 --- lib/tfw/config/__init__.py | 3 --- lib/tfw/config/envvars.py | 3 --- lib/tfw/crypto.py | 3 --- lib/tfw/decorators/__init__.py | 2 -- lib/tfw/decorators/lazy_property.py | 3 --- lib/tfw/decorators/rate_limiter.py | 3 --- lib/tfw/event_handlers/__init__.py | 3 --- lib/tfw/event_handlers/event_handler_base.py | 3 --- lib/tfw/fsm/__init__.py | 3 --- lib/tfw/fsm/fsm_base.py | 3 --- lib/tfw/fsm/linear_fsm.py | 3 --- lib/tfw/fsm/yaml_fsm.py | 3 --- lib/tfw/mixins/__init__.py | 2 -- lib/tfw/mixins/callback_mixin.py | 3 --- lib/tfw/mixins/supervisor_mixin.py | 3 --- lib/tfw/networking/__init__.py | 3 --- lib/tfw/networking/event_handler_connector.py | 3 --- lib/tfw/networking/serialization.py | 2 -- lib/tfw/networking/server_connector.py | 3 --- lib/tfw/server/tfw_server.py | 3 --- lib/tfw/server/zmq_websocket_router.py | 3 --- 43 files changed, 1 insertion(+), 131 deletions(-) delete mode 100644 LICENSE diff --git a/Dockerfile b/Dockerfile index 7bfe6e8..1290e10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,7 +54,7 @@ COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS} COPY nginx/nginx.conf ${TFW_NGINX_CONF} COPY nginx/default.conf ${TFW_NGINX_DEFAULT} COPY nginx/components/ ${TFW_NGINX_COMPONENTS} -COPY lib LICENSE ${TFW_LIB_DIR}/ +COPY lib ${TFW_LIB_DIR}/ COPY supervisor/tfw_server.py ${TFW_SERVER_DIR}/ RUN for dir in "${TFW_LIB_DIR}"/{tfw,tao,envvars} "/etc/nginx" "/etc/supervisor"; do \ diff --git a/LICENSE b/LICENSE deleted file mode 100644 index be3bf3b..0000000 --- a/LICENSE +++ /dev/null @@ -1,12 +0,0 @@ -AVATAO CONFIDENTIAL - -Copyright (C) 2018 Avatao.com Innovative Learning Kft. -All Rights Reserved. - -All source code, configuration files and documentation contained herein -is, and remains the exclusive property of Avatao.com Innovative Learning Kft. -The intellectual and technical concepts contained herein are proprietary -to Avatao.com Innovative Learning Kft. and are protected by trade secret -or copyright law. Dissemination of this information or reproduction of -this material is strictly forbidden unless prior written permission is -obtained from Avatao.com Innovative Learning Kft. diff --git a/lib/envvars/__init__.py b/lib/envvars/__init__.py index 46774d4..a96ed9f 100644 --- a/lib/envvars/__init__.py +++ b/lib/envvars/__init__.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from collections import namedtuple from os import environ diff --git a/lib/tao/__init__.py b/lib/tao/__init__.py index db64b25..e69de29 100644 --- a/lib/tao/__init__.py +++ b/lib/tao/__init__.py @@ -1,2 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. diff --git a/lib/tao/config/__init__.py b/lib/tao/config/__init__.py index 4876c01..fd07e8b 100644 --- a/lib/tao/config/__init__.py +++ b/lib/tao/config/__init__.py @@ -1,4 +1 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from .envvars import TAOENV diff --git a/lib/tao/config/envvars.py b/lib/tao/config/envvars.py index 260d057..813d06f 100644 --- a/lib/tao/config/envvars.py +++ b/lib/tao/config/envvars.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from envvars import LazyEnvironment TAOENV = LazyEnvironment('AVATAO_', 'taoenvtuple').environment diff --git a/lib/tfw/__init__.py b/lib/tfw/__init__.py index db64b25..e69de29 100644 --- a/lib/tfw/__init__.py +++ b/lib/tfw/__init__.py @@ -1,2 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. diff --git a/lib/tfw/builtins/directory_snapshotting_event_handler.py b/lib/tfw/builtins/directory_snapshotting_event_handler.py index 81683c5..25986eb 100644 --- a/lib/tfw/builtins/directory_snapshotting_event_handler.py +++ b/lib/tfw/builtins/directory_snapshotting_event_handler.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging from os.path import join as joinpath from os.path import basename diff --git a/lib/tfw/builtins/fsm_aware_event_handler.py b/lib/tfw/builtins/fsm_aware_event_handler.py index ac467ed..a619ca1 100644 --- a/lib/tfw/builtins/fsm_aware_event_handler.py +++ b/lib/tfw/builtins/fsm_aware_event_handler.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from abc import ABC from tfw.components import FSMAware diff --git a/lib/tfw/builtins/fsm_managing_event_handler.py b/lib/tfw/builtins/fsm_managing_event_handler.py index 8b5f4d6..d7bebdd 100644 --- a/lib/tfw/builtins/fsm_managing_event_handler.py +++ b/lib/tfw/builtins/fsm_managing_event_handler.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging from tfw.crypto import KeyManager, sign_message, verify_message diff --git a/lib/tfw/builtins/ide_event_handler.py b/lib/tfw/builtins/ide_event_handler.py index e03fc17..ab5a7eb 100644 --- a/lib/tfw/builtins/ide_event_handler.py +++ b/lib/tfw/builtins/ide_event_handler.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging from tfw.networking import Scope diff --git a/lib/tfw/builtins/log_monitoring_event_handler.py b/lib/tfw/builtins/log_monitoring_event_handler.py index 85699c8..9b9a900 100644 --- a/lib/tfw/builtins/log_monitoring_event_handler.py +++ b/lib/tfw/builtins/log_monitoring_event_handler.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging from tfw.networking import Scope diff --git a/lib/tfw/builtins/message_sender.py b/lib/tfw/builtins/message_sender.py index a4ee179..e6c1cfb 100644 --- a/lib/tfw/builtins/message_sender.py +++ b/lib/tfw/builtins/message_sender.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from .tfw_server_connector import TFWServerUplinkConnector diff --git a/lib/tfw/builtins/process_managing_event_handler.py b/lib/tfw/builtins/process_managing_event_handler.py index 5f687cd..abaec9d 100644 --- a/lib/tfw/builtins/process_managing_event_handler.py +++ b/lib/tfw/builtins/process_managing_event_handler.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging from xmlrpc.client import Fault as SupervisorFault diff --git a/lib/tfw/builtins/terminal_event_handler.py b/lib/tfw/builtins/terminal_event_handler.py index 9b0df08..2d45417 100644 --- a/lib/tfw/builtins/terminal_event_handler.py +++ b/lib/tfw/builtins/terminal_event_handler.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging from tfw.networking import Scope diff --git a/lib/tfw/components/__init__.py b/lib/tfw/components/__init__.py index 75e6531..99c5149 100644 --- a/lib/tfw/components/__init__.py +++ b/lib/tfw/components/__init__.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from .commands_equal import CommandsEqual from .file_manager import FileManager from .history_monitor import HistoryMonitor, BashMonitor, GDBMonitor diff --git a/lib/tfw/components/commands_equal.py b/lib/tfw/components/commands_equal.py index d2d4d15..b52aa6b 100644 --- a/lib/tfw/components/commands_equal.py +++ b/lib/tfw/components/commands_equal.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from shlex import split from re import search diff --git a/lib/tfw/components/fsm_aware.py b/lib/tfw/components/fsm_aware.py index beb4957..395a162 100644 --- a/lib/tfw/components/fsm_aware.py +++ b/lib/tfw/components/fsm_aware.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging from tfw.crypto import KeyManager, verify_message diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index 99add98..899217d 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from re import findall from re import compile as compileregex from abc import ABC, abstractmethod diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 8bd17bf..147fc2f 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import re from subprocess import run, CalledProcessError, PIPE from getpass import getuser diff --git a/lib/tfw/components/terminado_mini_server.py b/lib/tfw/components/terminado_mini_server.py index a440e23..b273f5b 100644 --- a/lib/tfw/components/terminado_mini_server.py +++ b/lib/tfw/components/terminado_mini_server.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging from tornado.ioloop import IOLoop diff --git a/lib/tfw/components/terminal_commands.py b/lib/tfw/components/terminal_commands.py index e536c92..adba1b4 100644 --- a/lib/tfw/components/terminal_commands.py +++ b/lib/tfw/components/terminal_commands.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging from abc import ABC from re import match diff --git a/lib/tfw/config/__init__.py b/lib/tfw/config/__init__.py index 3614ad9..b9719ef 100644 --- a/lib/tfw/config/__init__.py +++ b/lib/tfw/config/__init__.py @@ -1,4 +1 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from .envvars import TFWENV diff --git a/lib/tfw/config/envvars.py b/lib/tfw/config/envvars.py index ef590c6..68adb0e 100644 --- a/lib/tfw/config/envvars.py +++ b/lib/tfw/config/envvars.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from envvars import LazyEnvironment TFWENV = LazyEnvironment('TFW_', 'tfwenvtuple').environment diff --git a/lib/tfw/crypto.py b/lib/tfw/crypto.py index 344abee..bdd1fad 100644 --- a/lib/tfw/crypto.py +++ b/lib/tfw/crypto.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from functools import wraps from base64 import b64encode, b64decode from copy import deepcopy diff --git a/lib/tfw/decorators/__init__.py b/lib/tfw/decorators/__init__.py index db64b25..e69de29 100644 --- a/lib/tfw/decorators/__init__.py +++ b/lib/tfw/decorators/__init__.py @@ -1,2 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. diff --git a/lib/tfw/decorators/lazy_property.py b/lib/tfw/decorators/lazy_property.py index 14ad788..0bdd35f 100644 --- a/lib/tfw/decorators/lazy_property.py +++ b/lib/tfw/decorators/lazy_property.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from functools import update_wrapper, wraps diff --git a/lib/tfw/decorators/rate_limiter.py b/lib/tfw/decorators/rate_limiter.py index 6f666c8..5f8e253 100644 --- a/lib/tfw/decorators/rate_limiter.py +++ b/lib/tfw/decorators/rate_limiter.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from functools import wraps, partial from time import time, sleep diff --git a/lib/tfw/event_handlers/__init__.py b/lib/tfw/event_handlers/__init__.py index 447f617..7284ec8 100644 --- a/lib/tfw/event_handlers/__init__.py +++ b/lib/tfw/event_handlers/__init__.py @@ -1,4 +1 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from .event_handler_base import EventHandlerBase diff --git a/lib/tfw/event_handlers/event_handler_base.py b/lib/tfw/event_handlers/event_handler_base.py index 281c92f..ecf944b 100644 --- a/lib/tfw/event_handlers/event_handler_base.py +++ b/lib/tfw/event_handlers/event_handler_base.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging from inspect import currentframe from typing import Iterable diff --git a/lib/tfw/fsm/__init__.py b/lib/tfw/fsm/__init__.py index 11a7a58..2516525 100644 --- a/lib/tfw/fsm/__init__.py +++ b/lib/tfw/fsm/__init__.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from .fsm_base import FSMBase from .linear_fsm import LinearFSM from .yaml_fsm import YamlFSM diff --git a/lib/tfw/fsm/fsm_base.py b/lib/tfw/fsm/fsm_base.py index 5b050e0..e4f9645 100644 --- a/lib/tfw/fsm/fsm_base.py +++ b/lib/tfw/fsm/fsm_base.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging from collections import defaultdict from datetime import datetime diff --git a/lib/tfw/fsm/linear_fsm.py b/lib/tfw/fsm/linear_fsm.py index 5ac5eaf..383f98e 100644 --- a/lib/tfw/fsm/linear_fsm.py +++ b/lib/tfw/fsm/linear_fsm.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from transitions import State from tfw.fsm.fsm_base import FSMBase diff --git a/lib/tfw/fsm/yaml_fsm.py b/lib/tfw/fsm/yaml_fsm.py index fbfc542..facce7c 100644 --- a/lib/tfw/fsm/yaml_fsm.py +++ b/lib/tfw/fsm/yaml_fsm.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from subprocess import Popen, run from functools import partial, singledispatch from contextlib import suppress diff --git a/lib/tfw/mixins/__init__.py b/lib/tfw/mixins/__init__.py index db64b25..e69de29 100644 --- a/lib/tfw/mixins/__init__.py +++ b/lib/tfw/mixins/__init__.py @@ -1,2 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. diff --git a/lib/tfw/mixins/callback_mixin.py b/lib/tfw/mixins/callback_mixin.py index 8075f47..36b0f64 100644 --- a/lib/tfw/mixins/callback_mixin.py +++ b/lib/tfw/mixins/callback_mixin.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from functools import partial from tfw.decorators.lazy_property import lazy_property diff --git a/lib/tfw/mixins/supervisor_mixin.py b/lib/tfw/mixins/supervisor_mixin.py index 189d7cc..216f34b 100644 --- a/lib/tfw/mixins/supervisor_mixin.py +++ b/lib/tfw/mixins/supervisor_mixin.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import xmlrpc.client from xmlrpc.client import Fault as SupervisorFault from contextlib import suppress diff --git a/lib/tfw/networking/__init__.py b/lib/tfw/networking/__init__.py index 49e896d..62766cd 100644 --- a/lib/tfw/networking/__init__.py +++ b/lib/tfw/networking/__init__.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - from .serialization import serialize_tfw_msg, deserialize_tfw_msg, with_deserialize_tfw_msg, message_bytes from .server_connector import ServerUplinkConnector, ServerDownlinkConnector, ServerConnector from .event_handler_connector import EventHandlerConnector diff --git a/lib/tfw/networking/event_handler_connector.py b/lib/tfw/networking/event_handler_connector.py index e9b2511..60cbb94 100644 --- a/lib/tfw/networking/event_handler_connector.py +++ b/lib/tfw/networking/event_handler_connector.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging import zmq diff --git a/lib/tfw/networking/serialization.py b/lib/tfw/networking/serialization.py index c54609b..e329a6c 100644 --- a/lib/tfw/networking/serialization.py +++ b/lib/tfw/networking/serialization.py @@ -1,5 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. """ TFW JSON message format diff --git a/lib/tfw/networking/server_connector.py b/lib/tfw/networking/server_connector.py index dc3c78c..e1ef1fe 100644 --- a/lib/tfw/networking/server_connector.py +++ b/lib/tfw/networking/server_connector.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging from functools import partial diff --git a/lib/tfw/server/tfw_server.py b/lib/tfw/server/tfw_server.py index a7f7804..d524265 100644 --- a/lib/tfw/server/tfw_server.py +++ b/lib/tfw/server/tfw_server.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import logging from tornado.web import Application diff --git a/lib/tfw/server/zmq_websocket_router.py b/lib/tfw/server/zmq_websocket_router.py index 8ad0905..f682f02 100644 --- a/lib/tfw/server/zmq_websocket_router.py +++ b/lib/tfw/server/zmq_websocket_router.py @@ -1,6 +1,3 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - import json import logging From f94c5b0c37ebe7a649bba8159b4d5c9e84fded81 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 28 Jun 2019 17:34:22 +0200 Subject: [PATCH 060/174] Rename server_connector to uplink --- lib/tfw/builtins/message_sender.py | 6 +++--- lib/tfw/{config/log.py => logging.py} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename lib/tfw/{config/log.py => logging.py} (100%) diff --git a/lib/tfw/builtins/message_sender.py b/lib/tfw/builtins/message_sender.py index e6c1cfb..821b811 100644 --- a/lib/tfw/builtins/message_sender.py +++ b/lib/tfw/builtins/message_sender.py @@ -6,7 +6,7 @@ class MessageSender: Provides mechanisms to send messages to our frontend messaging component. """ def __init__(self): - self.server_connector = TFWServerUplinkConnector() + self.uplink = TFWServerUplinkConnector() self.key = 'message' self.queue_key = 'queueMessages' @@ -23,7 +23,7 @@ class MessageSender: 'message': message } } - self.server_connector.send_message(message) + self.uplink.send_message(message) def queue_messages(self, originator, messages): """ @@ -40,7 +40,7 @@ class MessageSender: ] } } - self.server_connector.send_message(message) + self.uplink.send_message(message) @staticmethod def generate_messages_from_queue(queue_message): diff --git a/lib/tfw/config/log.py b/lib/tfw/logging.py similarity index 100% rename from lib/tfw/config/log.py rename to lib/tfw/logging.py From 68a6469d2379ad290498e5bfdb29868a44bc2556 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 28 Jun 2019 17:34:37 +0200 Subject: [PATCH 061/174] Refactor TFW logging --- lib/tfw/logging.py | 89 +++++++++++++++++++++++++--------------- supervisor/tfw_server.py | 10 ++++- 2 files changed, 63 insertions(+), 36 deletions(-) diff --git a/lib/tfw/logging.py b/lib/tfw/logging.py index 1c01a7f..6969000 100644 --- a/lib/tfw/logging.py +++ b/lib/tfw/logging.py @@ -1,10 +1,9 @@ # pylint: disable=bad-whitespace -from sys import stderr from datetime import datetime +from typing import TextIO, Union +from dataclasses import dataclass from logging import DEBUG, getLogger, Handler, Formatter, Filter -from .envvars import TFWENV - class Color: GREY = '\033[30;1m' @@ -17,40 +16,55 @@ class Color: RESET = '\033[0m' -class TFWLog: - def __init__(self, path=TFWENV.LOGFILE, level=DEBUG): - self.log = getLogger() - self.old_level = self.log.level +@dataclass +class Log: + stream: Union[str, TextIO] + formatter: Formatter + + +class Logger: + def __init__(self, logs, level=DEBUG): + self.root_logger = getLogger() + self.old_level = self.root_logger.level self.new_level = level - self.handler = TFWLogHandler(path) - self.handler.setFormatter(TFWLogFormatter(20)) + self.handlers = [] + for log in logs: + handler = LogHandler(log.stream) + handler.setFormatter(log.formatter) + self.handlers.append(handler) def start(self): - self.log.setLevel(self.new_level) - self.log.addHandler(self.handler) + self.root_logger.setLevel(self.new_level) + for handler in self.handlers: + self.root_logger.addHandler(handler) def stop(self): - self.log.setLevel(self.old_level) - self.handler.close() - self.log.removeHandler(self.handler) + self.root_logger.setLevel(self.old_level) + for handler in self.handlers: + handler.close() + self.root_logger.removeHandler(handler) -class TFWLogHandler(Handler): - def __init__(self, path): - self.logfile = open(path, 'a+') +class LogHandler(Handler): + def __init__(self, stream): + if isinstance(stream, str): + self.stream = open(stream, 'a+') + self.close_stream = True + else: + self.stream = stream + self.close_stream = False super().__init__() def emit(self, record): - short_entry, long_entry = self.format(record) - stderr.write(short_entry+'\n') - self.logfile.write(long_entry+'\n') - stderr.flush() - self.logfile.flush() + entry = self.format(record) + self.stream.write(entry+'\n') + self.stream.flush() def close(self): - self.logfile.close() + if self.close_stream: + self.stream.close() -class TFWLogFormatter(Formatter): +class LogFormatter(Formatter): severity_to_color = { 'CRITICAL' : Color.BOLDRED, 'ERROR' : Color.RED, @@ -69,18 +83,13 @@ class TFWLogFormatter(Formatter): if record.args: tuple_args = (record.args,) if isinstance(record.args, dict) else record.args clean_args = tuple((self.trim(arg) for arg in tuple_args)) - short_message = record.msg % clean_args - long_message = record.msg % record.args + message = record.msg % clean_args else: - short_message = record.msg - long_message = record.msg + message = record.msg - short_entry = (f'[{Color.GREY}{date}{Color.RESET}|>' - f'{self.severity_to_color[record.levelname]}{record.module}:' - f'{record.levelname.lower()}{Color.RESET}] {short_message}') - long_entry = (f'[{date}|>{record.module}:{record.levelname.lower()}] ' - f'{long_message}') - return short_entry, long_entry + return (f'[{Color.GREY}{date}{Color.RESET}|>' + f'{self.severity_to_color[record.levelname]}{record.module}:' + f'{record.levelname.lower()}{Color.RESET}] {message}') def trim(self, value): if isinstance(value, dict): @@ -91,6 +100,18 @@ class TFWLogFormatter(Formatter): return value +class VerboseLogFormatter(Formatter): + def format(self, record): + date = datetime.utcfromtimestamp(record.created).strftime('%H:%M:%S') + if record.args: + message = record.msg % record.args + else: + message = record.msg + + return (f'[{date}|>{record.module}:{record.levelname.lower()}] ' + f'{message}') + + class WhitelistFilter(Filter): def __init__(self, names): self.names = names diff --git a/supervisor/tfw_server.py b/supervisor/tfw_server.py index 35fc2fa..812873c 100644 --- a/supervisor/tfw_server.py +++ b/supervisor/tfw_server.py @@ -1,10 +1,16 @@ +from sys import stderr + from tornado.ioloop import IOLoop -from tfw.config.log import TFWLog from tfw.server import TFWServer +from tfw.config import TFWENV +from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter if __name__ == '__main__': - TFWLog().start() + Logger([ + Log(stderr, LogFormatter(20)), + Log(TFWENV.LOGFILE, VerboseLogFormatter()) + ]).start() TFWServer().listen() IOLoop.instance().start() From 3be018eb17ab335ce3368aa5b6b1a03c9edbfe82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 1 Jul 2019 16:01:54 +0200 Subject: [PATCH 062/174] Make EventHandlerBase capable of stopping all it's instances --- lib/tfw/event_handlers/event_handler_base.py | 21 ++++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/tfw/event_handlers/event_handler_base.py b/lib/tfw/event_handlers/event_handler_base.py index ecf944b..051363e 100644 --- a/lib/tfw/event_handlers/event_handler_base.py +++ b/lib/tfw/event_handlers/event_handler_base.py @@ -1,5 +1,4 @@ import logging -from inspect import currentframe from typing import Iterable LOG = logging.getLogger(__name__) @@ -12,7 +11,10 @@ class EventHandlerBase: Derived classes must implement the handle_event() method """ + _instances = set() + def __init__(self, key, server_connector, scope): + type(self)._instances.add(self) self.server_connector = server_connector self.scope = scope self.keys = [] @@ -92,6 +94,11 @@ class EventHandlerBase: self.server_connector.unsubscribe(key) self.keys.remove(key) + @classmethod + def stop_all_instances(cls): + for instance in cls._instances: + instance.stop() + def stop(self): self.server_connector.close() self.cleanup() @@ -101,15 +108,3 @@ class EventHandlerBase: Perform cleanup actions such as releasing database connections and stuff like that. """ - - @classmethod - def get_local_instances(cls): - frame = currentframe() - if frame is None: - raise EnvironmentError('inspect.currentframe() is not supported!') - - locals_values = frame.f_back.f_locals.values() - return { - instance for instance in locals_values - if isinstance(instance, cls) - } From 882ab6049048d9b66370f746d021b8d67efbe8c1 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 2 Jul 2019 15:39:46 +0200 Subject: [PATCH 063/174] Print stack trace during an exception --- lib/tfw/logging.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/tfw/logging.py b/lib/tfw/logging.py index 6969000..584f257 100644 --- a/lib/tfw/logging.py +++ b/lib/tfw/logging.py @@ -2,6 +2,7 @@ from datetime import datetime from typing import TextIO, Union from dataclasses import dataclass +from traceback import format_exception from logging import DEBUG, getLogger, Handler, Formatter, Filter @@ -86,10 +87,11 @@ class LogFormatter(Formatter): message = record.msg % clean_args else: message = record.msg + trace = '\n'+''.join(format_exception(*record.exc_info)) if record.exc_info else '' return (f'[{Color.GREY}{date}{Color.RESET}|>' f'{self.severity_to_color[record.levelname]}{record.module}:' - f'{record.levelname.lower()}{Color.RESET}] {message}') + f'{record.levelname.lower()}{Color.RESET}] {message}{trace}') def trim(self, value): if isinstance(value, dict): @@ -107,9 +109,10 @@ class VerboseLogFormatter(Formatter): message = record.msg % record.args else: message = record.msg + trace = '\n'+''.join(format_exception(*record.exc_info)) if record.exc_info else '' return (f'[{date}|>{record.module}:{record.levelname.lower()}] ' - f'{message}') + f'{message}{trace}') class WhitelistFilter(Filter): From 81accf94ae5b5e4fb6281d28af2854be80dcce2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 2 Jul 2019 16:39:36 +0200 Subject: [PATCH 064/174] Fix inotify unit tests on BSD based systems --- lib/tfw/components/inotify/test_inotify.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/tfw/components/inotify/test_inotify.py b/lib/tfw/components/inotify/test_inotify.py index aa910fe..d132706 100644 --- a/lib/tfw/components/inotify/test_inotify.py +++ b/lib/tfw/components/inotify/test_inotify.py @@ -7,6 +7,7 @@ from shutil import rmtree from os.path import join from os import mkdir, remove, rename from tempfile import TemporaryDirectory +from contextlib import suppress import watchdog import pytest @@ -18,7 +19,8 @@ from .inotify import ( InotifyDirMovedEvent, InotifyDirDeletedEvent ) -watchdog.observers.inotify_buffer.InotifyBuffer.delay = 0 +with suppress(AttributeError): + watchdog.observers.inotify_buffer.InotifyBuffer.delay = 0 class InotifyContext: From 49844c7b581e6620c52d2aeff5c563ecd205054c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 3 Jul 2019 15:49:20 +0200 Subject: [PATCH 065/174] Fix trying to send None as a TFW message --- lib/tfw/builtins/fsm_managing_event_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/builtins/fsm_managing_event_handler.py b/lib/tfw/builtins/fsm_managing_event_handler.py index d7bebdd..fc3bcda 100644 --- a/lib/tfw/builtins/fsm_managing_event_handler.py +++ b/lib/tfw/builtins/fsm_managing_event_handler.py @@ -43,7 +43,7 @@ class FSMManagingEventHandler(EventHandler): sign_message(self.auth_key, message) sign_message(self.auth_key, fsm_update_message) self.server_connector.send_message(fsm_update_message, Scope.BROADCAST) - self.send_message(message) + self.send_message(message) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) From ca697b41b30625552d78e72879a823bcbf196a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 3 Jul 2019 16:48:17 +0200 Subject: [PATCH 066/174] Further improve EventHandler <-> EventHandlerBase port situation --- lib/tfw/builtins/event_handler.py | 5 ++- lib/tfw/builtins/fsm_aware_event_handler.py | 4 +-- lib/tfw/builtins/tfw_server_connector.py | 32 ++++++++++++-------- lib/tfw/event_handlers/event_handler_base.py | 13 ++++++-- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/lib/tfw/builtins/event_handler.py b/lib/tfw/builtins/event_handler.py index 1732044..9696fe4 100644 --- a/lib/tfw/builtins/event_handler.py +++ b/lib/tfw/builtins/event_handler.py @@ -1,10 +1,9 @@ from tfw.event_handlers import EventHandlerBase -from tfw.networking import Scope from .tfw_server_connector import TFWServerConnector class EventHandler(EventHandlerBase): # pylint: disable=abstract-method - def __init__(self, key, scope=Scope.ZMQ): - super().__init__(key, TFWServerConnector(), scope=scope) + def _build_server_connector(self): + return TFWServerConnector() diff --git a/lib/tfw/builtins/fsm_aware_event_handler.py b/lib/tfw/builtins/fsm_aware_event_handler.py index a619ca1..d585651 100644 --- a/lib/tfw/builtins/fsm_aware_event_handler.py +++ b/lib/tfw/builtins/fsm_aware_event_handler.py @@ -1,12 +1,10 @@ -from abc import ABC - from tfw.components import FSMAware from tfw.networking import Scope from .event_handler import EventHandler -class FSMAwareEventHandler(EventHandler, FSMAware, ABC): +class FSMAwareEventHandler(EventHandler, FSMAware): # pylint: disable=abstract-method """ Abstract base class for EventHandlers which automatically diff --git a/lib/tfw/builtins/tfw_server_connector.py b/lib/tfw/builtins/tfw_server_connector.py index 161a6cb..8b8b7c3 100644 --- a/lib/tfw/builtins/tfw_server_connector.py +++ b/lib/tfw/builtins/tfw_server_connector.py @@ -1,19 +1,25 @@ -from functools import partial - from tfw.networking import ServerUplinkConnector, ServerConnector from tfw.config import TFWENV -UPLINK_CONN_ADDR = f'tcp://localhost:{TFWENV.PULL_PORT}' -DOWNLINK_CONN_ADDR = f'tcp://localhost:{TFWENV.PUB_PORT}' +class ConnAddrMixin: + @property + def uplink_conn_addr(self): + return f'tcp://localhost:{TFWENV.PULL_PORT}' + + @property + def downlink_conn_addr(self): + return f'tcp://localhost:{TFWENV.PUB_PORT}' -TFWServerUplinkConnector = partial( - ServerUplinkConnector, - connect_addr=UPLINK_CONN_ADDR -) -TFWServerConnector = partial( - ServerConnector, - downlink_connect_addr=DOWNLINK_CONN_ADDR, - uplink_connect_addr=UPLINK_CONN_ADDR -) +class TFWServerUplinkConnector(ServerUplinkConnector, ConnAddrMixin): + def __init__(self): + super().__init__(self.uplink_conn_addr) + + +class TFWServerConnector(ServerConnector, ConnAddrMixin): + def __init__(self): + super().__init__( + self.downlink_conn_addr, + self.uplink_conn_addr + ) diff --git a/lib/tfw/event_handlers/event_handler_base.py b/lib/tfw/event_handlers/event_handler_base.py index 051363e..b87606f 100644 --- a/lib/tfw/event_handlers/event_handler_base.py +++ b/lib/tfw/event_handlers/event_handler_base.py @@ -1,10 +1,13 @@ import logging +from abc import ABC, abstractmethod from typing import Iterable +from tfw.networking import Scope + LOG = logging.getLogger(__name__) -class EventHandlerBase: +class EventHandlerBase(ABC): """ Abstract base class for all Python based EventHandlers. Useful implementation template for other languages. @@ -13,9 +16,9 @@ class EventHandlerBase: """ _instances = set() - def __init__(self, key, server_connector, scope): + def __init__(self, key, scope=Scope.ZMQ): type(self)._instances.add(self) - self.server_connector = server_connector + self.server_connector = self._build_server_connector() self.scope = scope self.keys = [] if isinstance(key, str): @@ -26,6 +29,10 @@ class EventHandlerBase: self.subscribe(*self.keys) self.server_connector.register_callback(self.event_handler_callback) + @abstractmethod + def _build_server_connector(self): + raise NotImplementedError() + def subscribe(self, *keys): """ Subscribe this EventHandler to receive events for given keys. From 459ae684de439b1edde7871bf7e578832a3baf7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 3 Jul 2019 16:48:56 +0200 Subject: [PATCH 067/174] Make up for forgotten parentheses --- lib/tfw/event_handlers/event_handler_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/event_handlers/event_handler_base.py b/lib/tfw/event_handlers/event_handler_base.py index b87606f..e56eef5 100644 --- a/lib/tfw/event_handlers/event_handler_base.py +++ b/lib/tfw/event_handlers/event_handler_base.py @@ -86,7 +86,7 @@ class EventHandlerBase(ABC): :param message: the message received :returns: the message to send back """ - raise NotImplementedError + raise NotImplementedError() def send_message(self, message): self.server_connector.send_message(message, self.scope) From 2c0aba2ffe63b0a046f1771690c9acc5e5e3dfae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 4 Jul 2019 15:19:48 +0200 Subject: [PATCH 068/174] Fix bug where sign_message wasn't idempotent --- lib/tfw/crypto.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tfw/crypto.py b/lib/tfw/crypto.py index bdd1fad..05f1daa 100644 --- a/lib/tfw/crypto.py +++ b/lib/tfw/crypto.py @@ -21,6 +21,7 @@ def message_checksum(message): def sign_message(key, message): + message.pop('signature', None) signature = message_signature(key, message) message['signature'] = b64encode(signature).decode() From 80e42a40916fcae64f896db99a0441b4f034a62a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 4 Jul 2019 15:51:55 +0200 Subject: [PATCH 069/174] Ignore message scope in message signatures --- lib/tfw/crypto.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/tfw/crypto.py b/lib/tfw/crypto.py index 05f1daa..562420b 100644 --- a/lib/tfw/crypto.py +++ b/lib/tfw/crypto.py @@ -21,6 +21,7 @@ def message_checksum(message): def sign_message(key, message): + message.pop('scope', None) message.pop('signature', None) signature = message_signature(key, message) message['signature'] = b64encode(signature).decode() @@ -31,6 +32,7 @@ def message_signature(key, message): def verify_message(key, message): + message.pop('scope', None) message = deepcopy(message) try: signature_b64 = message.pop('signature') From dc62508a7b0a7906ff673f5406a0ec20776e3983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 4 Jul 2019 15:55:23 +0200 Subject: [PATCH 070/174] Remove message scope during routing --- lib/tfw/server/zmq_websocket_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/server/zmq_websocket_router.py b/lib/tfw/server/zmq_websocket_router.py index f682f02..cb19c33 100644 --- a/lib/tfw/server/zmq_websocket_router.py +++ b/lib/tfw/server/zmq_websocket_router.py @@ -54,7 +54,7 @@ class TFWRouter: self.send_to_websockets = send_to_websockets def route(self, message): - scope = Scope(message.get('scope', 'zmq')) + scope = Scope(message.pop('scope', 'zmq')) routing_table = { Scope.ZMQ: self.send_to_zmq, From f6a369496dde35c502404945e17a75ef6c99ee9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 4 Jul 2019 17:27:25 +0200 Subject: [PATCH 071/174] Improve color scheme --- lib/tfw/logging.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/lib/tfw/logging.py b/lib/tfw/logging.py index 584f257..d405adb 100644 --- a/lib/tfw/logging.py +++ b/lib/tfw/logging.py @@ -7,14 +7,13 @@ from logging import DEBUG, getLogger, Handler, Formatter, Filter class Color: - GREY = '\033[30;1m' - RED = '\033[31m' - BOLDRED = '\033[31;1m' - BOLDGREEN = '\033[32;1m' - YELLOW = '\033[33;1m' - CYAN = '\033[36m' - BOLDWHITE = '\033[37;1m' - RESET = '\033[0m' + RED = '\033[31m' + GREEN = '\033[32m' + YELLOW = '\033[33m' + BLUE = '\033[34m' + CYAN = '\033[36m' + WHITE = '\033[37m' + RESET = '\033[0m' @dataclass @@ -67,11 +66,11 @@ class LogHandler(Handler): class LogFormatter(Formatter): severity_to_color = { - 'CRITICAL' : Color.BOLDRED, + 'CRITICAL' : Color.RED, 'ERROR' : Color.RED, 'WARNING' : Color.YELLOW, - 'INFO' : Color.BOLDGREEN, - 'DEBUG' : Color.BOLDWHITE, + 'INFO' : Color.GREEN, + 'DEBUG' : Color.BLUE, 'NOTSET' : Color.CYAN } @@ -80,7 +79,7 @@ class LogFormatter(Formatter): super().__init__() def format(self, record): - date = datetime.utcfromtimestamp(record.created).strftime('%H:%M:%S') + time = datetime.utcfromtimestamp(record.created).strftime('%H:%M:%S') if record.args: tuple_args = (record.args,) if isinstance(record.args, dict) else record.args clean_args = tuple((self.trim(arg) for arg in tuple_args)) @@ -89,7 +88,7 @@ class LogFormatter(Formatter): message = record.msg trace = '\n'+''.join(format_exception(*record.exc_info)) if record.exc_info else '' - return (f'[{Color.GREY}{date}{Color.RESET}|>' + return (f'[{Color.WHITE}{time}{Color.RESET}|>' f'{self.severity_to_color[record.levelname]}{record.module}:' f'{record.levelname.lower()}{Color.RESET}] {message}{trace}') From fbe60de968e961a6dbb3e6200675d797823cb202 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 5 Jul 2019 15:25:59 +0200 Subject: [PATCH 072/174] Separate independent classes from built-in event handlers --- lib/tfw/builtins/__init__.py | 7 ++- lib/tfw/builtins/frontend_event_handler.py | 45 +---------------- .../builtins/fsm_managing_event_handler.py | 28 +---------- .../builtins/log_monitoring_event_handler.py | 49 +++---------------- .../process_managing_event_handler.py | 33 ++++++------- lib/tfw/builtins/terminal_event_handler.py | 3 +- lib/tfw/components/__init__.py | 12 +++-- lib/tfw/components/fsm_updater.py | 25 ++++++++++ lib/tfw/components/log_inotify_observer.py | 45 +++++++++++++++++ .../message_sender.py | 7 +-- lib/tfw/components/message_storage.py | 44 +++++++++++++++++ .../supervisor.py} | 19 +++---- 12 files changed, 160 insertions(+), 157 deletions(-) create mode 100644 lib/tfw/components/fsm_updater.py create mode 100644 lib/tfw/components/log_inotify_observer.py rename lib/tfw/{builtins => components}/message_sender.py (91%) create mode 100644 lib/tfw/components/message_storage.py rename lib/tfw/{mixins/supervisor_mixin.py => components/supervisor.py} (73%) diff --git a/lib/tfw/builtins/__init__.py b/lib/tfw/builtins/__init__.py index af322b8..c8ab18b 100644 --- a/lib/tfw/builtins/__init__.py +++ b/lib/tfw/builtins/__init__.py @@ -1,13 +1,12 @@ -from .tfw_server_connector import TFWServerUplinkConnector, TFWServerConnector -from .event_handler import EventHandler -from .fsm_aware_event_handler import FSMAwareEventHandler from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler +from .event_handler import EventHandler from .frontend_event_handler import FrontendEventHandler +from .fsm_aware_event_handler import FSMAwareEventHandler from .fsm_managing_event_handler import FSMManagingEventHandler from .ide_event_handler import IdeEventHandler from .log_monitoring_event_handler import LogMonitoringEventHandler -from .message_sender import MessageSender from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler from .process_managing_event_handler import ProcessManagingEventHandler from .terminal_event_handler import TerminalEventHandler +from .tfw_server_connector import TFWServerUplinkConnector, TFWServerConnector diff --git a/lib/tfw/builtins/frontend_event_handler.py b/lib/tfw/builtins/frontend_event_handler.py index 1099564..ba39c27 100644 --- a/lib/tfw/builtins/frontend_event_handler.py +++ b/lib/tfw/builtins/frontend_event_handler.py @@ -1,9 +1,6 @@ -from abc import ABC, abstractmethod -from contextlib import suppress - from tfw.networking import Scope +from tfw.components import FrontendMessageStorage -from .message_sender import MessageSender from .event_handler import EventHandler @@ -22,43 +19,3 @@ class FrontendEventHandler(EventHandler): def recover_frontend(self): for message in self._frontend_message_storage.messages: self.send_message(message) - - -class MessageStorage(ABC): - def __init__(self): - self._messages = [] - - def save_message(self, message): - with suppress(KeyError, AttributeError): - if self._filter_message(message): - self._messages.extend(self._transform_message(message)) - - @abstractmethod - def _filter_message(self, message): - raise NotImplementedError - - def _transform_message(self, message): # pylint: disable=no-self-use - yield message - - def clear(self): - self._messages.clear() - - @property - def messages(self): - yield from self._messages - - -class FrontendMessageStorage(MessageStorage): - def __init__(self, keys): - self._keys = keys - super().__init__() - - def _filter_message(self, message): - key = message['key'] - return key in self._keys - - def _transform_message(self, message): - if message['key'] == 'queueMessages': - yield from MessageSender.generate_messages_from_queue(message) - else: - yield message diff --git a/lib/tfw/builtins/fsm_managing_event_handler.py b/lib/tfw/builtins/fsm_managing_event_handler.py index fc3bcda..ccbb7de 100644 --- a/lib/tfw/builtins/fsm_managing_event_handler.py +++ b/lib/tfw/builtins/fsm_managing_event_handler.py @@ -2,6 +2,7 @@ import logging from tfw.crypto import KeyManager, sign_message, verify_message from tfw.networking import Scope +from tfw.components import FSMUpdater from .event_handler import EventHandler @@ -69,30 +70,3 @@ class FSMManagingEventHandler(EventHandler): """ # pylint: disable=no-self-use return message - - -class FSMUpdater: - def __init__(self, fsm): - self.fsm = fsm - - @property - def fsm_update(self): - return { - 'key': 'fsm_update', - 'data': self.fsm_update_data - } - - @property - def fsm_update_data(self): - valid_transitions = [ - {'trigger': trigger} - for trigger in self.fsm.get_triggers(self.fsm.state) - ] - last_fsm_event = self.fsm.event_log[-1] - last_fsm_event['timestamp'] = last_fsm_event['timestamp'].isoformat() - return { - 'current_state': self.fsm.state, - 'valid_transitions': valid_transitions, - 'in_accepted_state': self.fsm.in_accepted_state, - 'last_event': last_fsm_event - } diff --git a/lib/tfw/builtins/log_monitoring_event_handler.py b/lib/tfw/builtins/log_monitoring_event_handler.py index 9b9a900..eebade5 100644 --- a/lib/tfw/builtins/log_monitoring_event_handler.py +++ b/lib/tfw/builtins/log_monitoring_event_handler.py @@ -1,52 +1,14 @@ import logging +from tfw.config import TFWENV from tfw.networking import Scope -from tfw.components.inotify import InotifyObserver -from tfw.mixins.supervisor_mixin import SupervisorLogMixin +from tfw.components import LogInotifyObserver from .event_handler import EventHandler LOG = logging.getLogger(__name__) -class LogInotifyObserver(InotifyObserver, SupervisorLogMixin): - def __init__(self, server_connector, process_name, log_tail=0): - self._prevent_log_recursion() - self._server_connector = server_connector - self._process_name = process_name - self.log_tail = log_tail - self._procinfo = None - InotifyObserver.__init__(self, self._get_logfiles()) - - @staticmethod - def _prevent_log_recursion(): - # This is done to prevent inotify event logs triggering themselves (infinite log recursion) - logging.getLogger('watchdog.observers.inotify_buffer').propagate = False - - def _get_logfiles(self): - self._procinfo = self.supervisor.getProcessInfo(self._process_name) - return self._procinfo['stdout_logfile'], self._procinfo['stderr_logfile'] - - @property - def process_name(self): - return self._process_name - - @process_name.setter - def process_name(self, process_name): - self._process_name = process_name - self.paths = self._get_logfiles() - - def on_modified(self, event): - self._server_connector.send_message({ - 'key': 'processlog', - 'data': { - 'command': 'new_log', - 'stdout': self.read_stdout(self.process_name, tail=self.log_tail), - 'stderr': self.read_stderr(self.process_name, tail=self.log_tail) - } - }, Scope.BROADCAST) - - class LogMonitoringEventHandler(EventHandler): """ Monitors the output of a supervisor process (stdout, stderr) and @@ -60,7 +22,12 @@ class LogMonitoringEventHandler(EventHandler): def __init__(self, key, process_name, log_tail=0): super().__init__(key, scope=Scope.WEBSOCKET) self.process_name = process_name - self._monitor = LogInotifyObserver(self.server_connector, process_name, log_tail) + self._monitor = LogInotifyObserver( + server_connector=self.server_connector, + supervisor_uri=TFWENV.SUPERVISOR_HTTP_URI, + process_name=process_name, + log_tail=log_tail + ) self._monitor.start() self.command_handlers = { diff --git a/lib/tfw/builtins/process_managing_event_handler.py b/lib/tfw/builtins/process_managing_event_handler.py index abaec9d..2a1f968 100644 --- a/lib/tfw/builtins/process_managing_event_handler.py +++ b/lib/tfw/builtins/process_managing_event_handler.py @@ -1,27 +1,16 @@ import logging from xmlrpc.client import Fault as SupervisorFault +from tfw.config import TFWENV from tfw.networking import Scope -from tfw.mixins.supervisor_mixin import SupervisorMixin, SupervisorLogMixin +from tfw.components import ProcessManager, LogManager from .event_handler import EventHandler LOG = logging.getLogger(__name__) -class ProcessManager(SupervisorMixin, SupervisorLogMixin): - def __init__(self): - self.commands = { - 'start': self.start_process, - 'stop': self.stop_process, - 'restart': self.restart_process - } - - def __call__(self, command, process_name): - return self.commands[command](process_name) - - -class ProcessManagingEventHandler(EventHandler): +class ProcessManagingEventHandler(EventHandler, ProcessManager, LogManager): """ Event handler that can manage processes managed by supervisor. @@ -35,23 +24,29 @@ class ProcessManagingEventHandler(EventHandler): (the names are as self-documenting as it gets) """ def __init__(self, key, log_tail=0): - super().__init__(key, scope=Scope.WEBSOCKET) - self.processmanager = ProcessManager() + EventHandler.__init__(self, key, scope=Scope.WEBSOCKET) + ProcessManager.__init__(self, TFWENV.SUPERVISOR_HTTP_URI) + LogManager.__init__(self, TFWENV.SUPERVISOR_HTTP_URI) self.log_tail = log_tail + self.commands = { + 'start': self.start_process, + 'stop': self.stop_process, + 'restart': self.restart_process + } def handle_event(self, message): try: data = message['data'] try: - self.processmanager(data['command'], data['process_name']) + self.commands[data['command']](data['process_name']) except SupervisorFault as fault: message['data']['error'] = fault.faultString finally: - message['data']['stdout'] = self.processmanager.read_stdout( + message['data']['stdout'] = self.read_stdout( data['process_name'], self.log_tail ) - message['data']['stderr'] = self.processmanager.read_stderr( + message['data']['stderr'] = self.read_stderr( data['process_name'], self.log_tail ) diff --git a/lib/tfw/builtins/terminal_event_handler.py b/lib/tfw/builtins/terminal_event_handler.py index 2d45417..1f7bf5f 100644 --- a/lib/tfw/builtins/terminal_event_handler.py +++ b/lib/tfw/builtins/terminal_event_handler.py @@ -1,8 +1,7 @@ import logging from tfw.networking import Scope -from tfw.components import BashMonitor -from tfw.components.terminado_mini_server import TerminadoMiniServer +from tfw.components import BashMonitor, TerminadoMiniServer from tfw.config import TFWENV from tao.config import TAOENV diff --git a/lib/tfw/components/__init__.py b/lib/tfw/components/__init__.py index 99c5149..a1355d7 100644 --- a/lib/tfw/components/__init__.py +++ b/lib/tfw/components/__init__.py @@ -1,6 +1,12 @@ from .commands_equal import CommandsEqual from .file_manager import FileManager -from .history_monitor import HistoryMonitor, BashMonitor, GDBMonitor -from .snapshot_provider import SnapshotProvider -from .terminal_commands import TerminalCommands from .fsm_aware import FSMAware +from .fsm_updater import FSMUpdater +from .history_monitor import BashMonitor, GDBMonitor +from .log_inotify_observer import LogInotifyObserver +from .message_sender import MessageSender +from .message_storage import FrontendMessageStorage +from .snapshot_provider import SnapshotProvider +from .supervisor import ProcessManager, LogManager +from .terminado_mini_server import TerminadoMiniServer +from .terminal_commands import TerminalCommands diff --git a/lib/tfw/components/fsm_updater.py b/lib/tfw/components/fsm_updater.py new file mode 100644 index 0000000..6bff16a --- /dev/null +++ b/lib/tfw/components/fsm_updater.py @@ -0,0 +1,25 @@ +class FSMUpdater: + def __init__(self, fsm): + self.fsm = fsm + + @property + def fsm_update(self): + return { + 'key': 'fsm_update', + 'data': self.fsm_update_data + } + + @property + def fsm_update_data(self): + valid_transitions = [ + {'trigger': trigger} + for trigger in self.fsm.get_triggers(self.fsm.state) + ] + last_fsm_event = self.fsm.event_log[-1] + last_fsm_event['timestamp'] = last_fsm_event['timestamp'].isoformat() + return { + 'current_state': self.fsm.state, + 'valid_transitions': valid_transitions, + 'in_accepted_state': self.fsm.in_accepted_state, + 'last_event': last_fsm_event + } diff --git a/lib/tfw/components/log_inotify_observer.py b/lib/tfw/components/log_inotify_observer.py new file mode 100644 index 0000000..4a21f08 --- /dev/null +++ b/lib/tfw/components/log_inotify_observer.py @@ -0,0 +1,45 @@ +import logging + +from tfw.networking import Scope + +from .inotify import InotifyObserver +from .supervisor import LogManager + + +class LogInotifyObserver(InotifyObserver, LogManager): + def __init__(self, server_connector, supervisor_uri, process_name, log_tail=0): + self._prevent_log_recursion() + self._server_connector = server_connector + self._process_name = process_name + self.log_tail = log_tail + self._procinfo = None + LogManager.__init__(self, supervisor_uri) + InotifyObserver.__init__(self, self._get_logfiles()) + + @staticmethod + def _prevent_log_recursion(): + # This is done to prevent inotify event logs triggering themselves (infinite log recursion) + logging.getLogger('watchdog.observers.inotify_buffer').propagate = False + + def _get_logfiles(self): + self._procinfo = self.supervisor.getProcessInfo(self._process_name) + return self._procinfo['stdout_logfile'], self._procinfo['stderr_logfile'] + + @property + def process_name(self): + return self._process_name + + @process_name.setter + def process_name(self, process_name): + self._process_name = process_name + self.paths = self._get_logfiles() + + def on_modified(self, event): + self._server_connector.send_message({ + 'key': 'processlog', + 'data': { + 'command': 'new_log', + 'stdout': self.read_stdout(self.process_name, tail=self.log_tail), + 'stderr': self.read_stderr(self.process_name, tail=self.log_tail) + } + }, Scope.BROADCAST) diff --git a/lib/tfw/builtins/message_sender.py b/lib/tfw/components/message_sender.py similarity index 91% rename from lib/tfw/builtins/message_sender.py rename to lib/tfw/components/message_sender.py index 821b811..72915c2 100644 --- a/lib/tfw/builtins/message_sender.py +++ b/lib/tfw/components/message_sender.py @@ -1,12 +1,9 @@ -from .tfw_server_connector import TFWServerUplinkConnector - - class MessageSender: """ Provides mechanisms to send messages to our frontend messaging component. """ - def __init__(self): - self.uplink = TFWServerUplinkConnector() + def __init__(self, uplink): + self.uplink = uplink self.key = 'message' self.queue_key = 'queueMessages' diff --git a/lib/tfw/components/message_storage.py b/lib/tfw/components/message_storage.py new file mode 100644 index 0000000..1c63ffc --- /dev/null +++ b/lib/tfw/components/message_storage.py @@ -0,0 +1,44 @@ +from abc import ABC, abstractmethod +from contextlib import suppress + +from .message_sender import MessageSender + + +class MessageStorage(ABC): + def __init__(self): + self._messages = [] + + def save_message(self, message): + with suppress(KeyError, AttributeError): + if self._filter_message(message): + self._messages.extend(self._transform_message(message)) + + @abstractmethod + def _filter_message(self, message): + raise NotImplementedError + + def _transform_message(self, message): # pylint: disable=no-self-use + yield message + + def clear(self): + self._messages.clear() + + @property + def messages(self): + yield from self._messages + + +class FrontendMessageStorage(MessageStorage): + def __init__(self, keys): + self._keys = keys + super().__init__() + + def _filter_message(self, message): + key = message['key'] + return key in self._keys + + def _transform_message(self, message): + if message['key'] == 'queueMessages': + yield from MessageSender.generate_messages_from_queue(message) + else: + yield message diff --git a/lib/tfw/mixins/supervisor_mixin.py b/lib/tfw/components/supervisor.py similarity index 73% rename from lib/tfw/mixins/supervisor_mixin.py rename to lib/tfw/components/supervisor.py index 216f34b..83e8bc4 100644 --- a/lib/tfw/mixins/supervisor_mixin.py +++ b/lib/tfw/components/supervisor.py @@ -1,20 +1,15 @@ +from os import remove +from contextlib import suppress import xmlrpc.client from xmlrpc.client import Fault as SupervisorFault -from contextlib import suppress -from os import remove - -from tfw.decorators.lazy_property import lazy_property -from tfw.config import TFWENV -class SupervisorBaseMixin: - @lazy_property - def supervisor(self): - # pylint: disable=no-self-use - return xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor +class SupervisorBase: + def __init__(self, supervisor_uri): + self.supervisor = xmlrpc.client.ServerProxy(supervisor_uri).supervisor -class SupervisorMixin(SupervisorBaseMixin): +class ProcessManager(SupervisorBase): def stop_process(self, process_name): with suppress(SupervisorFault): self.supervisor.stopProcess(process_name) @@ -27,7 +22,7 @@ class SupervisorMixin(SupervisorBaseMixin): self.start_process(process_name) -class SupervisorLogMixin(SupervisorBaseMixin): +class LogManager(SupervisorBase): def read_stdout(self, process_name, tail=0): return self.supervisor.readProcessStdoutLog(process_name, -tail, 0) From 98271a37838b47e5a411186048d6247eb24ccbeb Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 8 Jul 2019 11:44:39 +0200 Subject: [PATCH 073/174] Remove dead code --- lib/tfw/components/terminado_mini_server.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/lib/tfw/components/terminado_mini_server.py b/lib/tfw/components/terminado_mini_server.py index b273f5b..dd43ced 100644 --- a/lib/tfw/components/terminado_mini_server.py +++ b/lib/tfw/components/terminado_mini_server.py @@ -1,11 +1,8 @@ import logging -from tornado.ioloop import IOLoop from tornado.web import Application from terminado import TermSocket, SingleTermManager -from tfw.config import TFWENV - LOG = logging.getLogger(__name__) @@ -45,14 +42,3 @@ class TerminadoMiniServer: def stop(self): self.term_manager.shutdown() - - -if __name__ == '__main__': - LOG.info('Terminado Mini Server listening on %s', TFWENV.TERMINADO_PORT) - TerminadoMiniServer( - '/terminal', - TFWENV.TERMINADO_PORT, - TFWENV.TERMINADO_WD, - ['bash'] - ).listen() - IOLoop.instance().start() From 9be21d88ac500e10ca91dd95f5b1f3b7218cac7d Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 8 Jul 2019 14:06:44 +0200 Subject: [PATCH 074/174] Create event handler to wrap TerminalCommands --- lib/tfw/builtins/__init__.py | 1 + .../builtins/terminal_commands_event_handler.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 lib/tfw/builtins/terminal_commands_event_handler.py diff --git a/lib/tfw/builtins/__init__.py b/lib/tfw/builtins/__init__.py index c8ab18b..d276619 100644 --- a/lib/tfw/builtins/__init__.py +++ b/lib/tfw/builtins/__init__.py @@ -8,5 +8,6 @@ from .log_monitoring_event_handler import LogMonitoringEventHandler from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler from .process_managing_event_handler import ProcessManagingEventHandler +from .terminal_commands_event_handler import TerminalCommandsEventHandler from .terminal_event_handler import TerminalEventHandler from .tfw_server_connector import TFWServerUplinkConnector, TFWServerConnector diff --git a/lib/tfw/builtins/terminal_commands_event_handler.py b/lib/tfw/builtins/terminal_commands_event_handler.py new file mode 100644 index 0000000..0385f0b --- /dev/null +++ b/lib/tfw/builtins/terminal_commands_event_handler.py @@ -0,0 +1,14 @@ +from tfw.components import TerminalCommands +from tfw.networking import Scope + +from .event_handler import EventHandler + + +class TerminalCommandsEventHandler(EventHandler, TerminalCommands): + def __init__(self, key, scope=Scope.ZMQ, bashrc=None): + EventHandler.__init__(self, key, scope) + TerminalCommands.__init__(self, bashrc) + + def handle_event(self, message): + command = message['value'] + self.callback(command) From 64ea7d8218523779c64e66d7a55e6017ea22c790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 8 Jul 2019 20:12:12 +0200 Subject: [PATCH 075/174] Fix ZMQ socketopt call order --- lib/tfw/networking/server_connector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tfw/networking/server_connector.py b/lib/tfw/networking/server_connector.py index e1ef1fe..88e9450 100644 --- a/lib/tfw/networking/server_connector.py +++ b/lib/tfw/networking/server_connector.py @@ -13,8 +13,8 @@ LOG = logging.getLogger(__name__) class ServerDownlinkConnector(): def __init__(self, connect_addr): self._zmq_sub_socket = zmq.Context.instance().socket(zmq.SUB) - self._zmq_sub_socket.connect(connect_addr) self._zmq_sub_socket.setsockopt(zmq.RCVHWM, 0) + self._zmq_sub_socket.connect(connect_addr) self._zmq_sub_stream = ZMQStream(self._zmq_sub_socket) self.subscribe = partial(self._zmq_sub_socket.setsockopt_string, zmq.SUBSCRIBE) @@ -31,8 +31,8 @@ class ServerDownlinkConnector(): class ServerUplinkConnector(): def __init__(self, connect_addr): self._zmq_push_socket = zmq.Context.instance().socket(zmq.PUSH) - self._zmq_push_socket.connect(connect_addr) self._zmq_push_socket.setsockopt(zmq.SNDHWM, 0) + self._zmq_push_socket.connect(connect_addr) def send_message(self, message, scope=Scope.ZMQ): message['scope'] = scope.value From e5dee3a1b3b453ed406d6f27ddeace80cffff0fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 9 Jul 2019 09:12:21 +0200 Subject: [PATCH 076/174] Remove leftover parentheses --- lib/tfw/networking/event_handler_connector.py | 4 ++-- lib/tfw/networking/server_connector.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/tfw/networking/event_handler_connector.py b/lib/tfw/networking/event_handler_connector.py index 60cbb94..74b6270 100644 --- a/lib/tfw/networking/event_handler_connector.py +++ b/lib/tfw/networking/event_handler_connector.py @@ -8,7 +8,7 @@ from .serialization import serialize_tfw_msg, with_deserialize_tfw_msg LOG = logging.getLogger(__name__) -class EventHandlerDownlinkConnector(): +class EventHandlerDownlinkConnector: def __init__(self, bind_addr): self._zmq_pull_socket = zmq.Context.instance().socket(zmq.PULL) self._zmq_pull_socket.setsockopt(zmq.RCVHWM, 0) @@ -24,7 +24,7 @@ class EventHandlerDownlinkConnector(): self._zmq_pull_stream.close() -class EventHandlerUplinkConnector(): +class EventHandlerUplinkConnector: def __init__(self, bind_addr): self._zmq_pub_socket = zmq.Context.instance().socket(zmq.PUB) self._zmq_pub_socket.setsockopt(zmq.SNDHWM, 0) diff --git a/lib/tfw/networking/server_connector.py b/lib/tfw/networking/server_connector.py index 88e9450..da407ac 100644 --- a/lib/tfw/networking/server_connector.py +++ b/lib/tfw/networking/server_connector.py @@ -10,7 +10,7 @@ from .serialization import serialize_tfw_msg, with_deserialize_tfw_msg LOG = logging.getLogger(__name__) -class ServerDownlinkConnector(): +class ServerDownlinkConnector: def __init__(self, connect_addr): self._zmq_sub_socket = zmq.Context.instance().socket(zmq.SUB) self._zmq_sub_socket.setsockopt(zmq.RCVHWM, 0) @@ -28,7 +28,7 @@ class ServerDownlinkConnector(): self._zmq_sub_stream.close() -class ServerUplinkConnector(): +class ServerUplinkConnector: def __init__(self, connect_addr): self._zmq_push_socket = zmq.Context.instance().socket(zmq.PUSH) self._zmq_push_socket.setsockopt(zmq.SNDHWM, 0) From 84a28a1582eea4dec0e8167967dcfb13fb761b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 9 Jul 2019 11:02:45 +0200 Subject: [PATCH 077/174] Remove unused rate limiter classes (sad to see you go) --- lib/tfw/decorators/rate_limiter.py | 98 ------------------------------ 1 file changed, 98 deletions(-) delete mode 100644 lib/tfw/decorators/rate_limiter.py diff --git a/lib/tfw/decorators/rate_limiter.py b/lib/tfw/decorators/rate_limiter.py deleted file mode 100644 index 5f8e253..0000000 --- a/lib/tfw/decorators/rate_limiter.py +++ /dev/null @@ -1,98 +0,0 @@ -from functools import wraps, partial -from time import time, sleep - -from tfw.decorators.lazy_property import lazy_property - - -class RateLimiter: - """ - Decorator class for rate limiting, blocking. - - When applied to a function this decorator will apply rate limiting - if the function is invoked more frequently than rate_per_seconds. - - By default rate limiting means sleeping until the next invocation time - as per __init__ parameter rate_per_seconds. - - Note that this decorator BLOCKS THE THREAD it is being executed on, - so it is only acceptable for stuff running on a separate thread. - - If this is no good for you please refer to AsyncRateLimiter in this module, - which is designed not to block and use the IOLoop it is being called from. - """ - def __init__(self, rate_per_second): - """ - :param rate_per_second: max frequency the decorated method should be - invoked with - """ - self.min_interval = 1 / float(rate_per_second) - self.fun = None - self.last_call = time() - - def action(self, seconds_to_next_call): - if seconds_to_next_call: - sleep(seconds_to_next_call) - self.fun() - - def __call__(self, fun): - @wraps(fun) - def wrapper(*args, **kwargs): - self.fun = partial(fun, *args, **kwargs) - limit_seconds = self._limit_rate() - self.action(limit_seconds) - return wrapper - - def _limit_rate(self): - seconds_since_last_call = time() - self.last_call - seconds_to_next_call = self.min_interval - seconds_since_last_call - - if seconds_to_next_call > 0: - return seconds_to_next_call - self.last_call = time() - return 0 - - -class AsyncRateLimiter(RateLimiter): - """ - Decorator class for rate limiting, non-blocking. - - The semantics of the rate limiting: - - unlike RateLimiter this decorator never blocks, instead it adds an async - callback version of the decorated function to the IOLoop - (to be executed after the rate limiting has expired). - - the timing works similarly to RateLimiter - """ - def __init__(self, rate_per_second, ioloop_factory): - """ - :param rate_per_second: max frequency the decorated method should be - invoked with - :param ioloop_factory: callable that should return an instance of the - IOLoop of the application - """ - self._ioloop_factory = ioloop_factory - self._ioloop = None - self._last_callback = None - - self._make_action_thread_safe() - super().__init__(rate_per_second=rate_per_second) - - def _make_action_thread_safe(self): - self.action = partial(self.ioloop.add_callback, self.action) - - @lazy_property - def ioloop(self): - return self._ioloop_factory() - - def action(self, seconds_to_next_call): - # pylint: disable=method-hidden - if self._last_callback: - self.ioloop.remove_timeout(self._last_callback) - - self._last_callback = self.ioloop.call_later( - seconds_to_next_call, - self.fun_with_debounce - ) - - def fun_with_debounce(self): - self.last_call = time() - self.fun() From f1679ffb502cfbd068b89713768716cfe1d27324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 10 Jul 2019 14:26:12 +0200 Subject: [PATCH 078/174] Add new EventHandler stuff as per interface segregation principle --- lib/tfw/event_handlers/event_handler.py | 27 ++++ .../event_handlers/event_handler_factory.py | 51 ++++++ lib/tfw/event_handlers/test_event_handler.py | 147 ++++++++++++++++++ 3 files changed, 225 insertions(+) create mode 100644 lib/tfw/event_handlers/event_handler.py create mode 100644 lib/tfw/event_handlers/event_handler_factory.py create mode 100644 lib/tfw/event_handlers/test_event_handler.py diff --git a/lib/tfw/event_handlers/event_handler.py b/lib/tfw/event_handlers/event_handler.py new file mode 100644 index 0000000..a75381e --- /dev/null +++ b/lib/tfw/event_handlers/event_handler.py @@ -0,0 +1,27 @@ +class EventHandler: + _instances = set() + + def __init__(self, server_connector): + type(self)._instances.add(self) + self.server_connector = server_connector + + def start(self): + self.server_connector.register_callback(self._event_callback) + + def _event_callback(self, message): + self.handle_event(message, self.server_connector) + + def handle_event(self, message, server_connector): + raise NotImplementedError() + + @classmethod + def stop_all_instances(cls): + for instance in cls._instances: + instance.stop() + + def stop(self): + self.server_connector.close() + self.cleanup() + + def cleanup(self): + pass diff --git a/lib/tfw/event_handlers/event_handler_factory.py b/lib/tfw/event_handlers/event_handler_factory.py new file mode 100644 index 0000000..c3a6b18 --- /dev/null +++ b/lib/tfw/event_handlers/event_handler_factory.py @@ -0,0 +1,51 @@ +from contextlib import suppress + +from .event_handler import EventHandler + + +class EventHandlerFactoryBase: + def build(self, event_handler, *, keys=None): + analyzer = EventHandlerAnalyzer(event_handler, keys) + event_handler = self._build_from_callable(analyzer) + event_handler.start() + return event_handler + + def _build_from_callable(self, analyzer): + server_connector = self._build_server_connector() + server_connector.subscribe(analyzer.keys) + event_handler = EventHandler(server_connector) + event_handler.handle_event = analyzer.handle_event + with suppress(AttributeError): + event_handler.cleanup = analyzer.cleanup + return event_handler + + def _build_server_connector(self): + raise NotImplementedError() + + +class EventHandlerAnalyzer: + def __init__(self, event_handler, supplied_keys): + self._event_handler = event_handler + self._supplied_keys = supplied_keys + + @property + def keys(self): + if self._supplied_keys is None: + try: + return self._event_handler.keys + except AttributeError: + raise ValueError('No keys supplied!') + return self._supplied_keys + + @property + def handle_event(self): + try: + return self._event_handler.handle_event + except AttributeError: + if callable(self._event_handler): + return self._event_handler + raise ValueError('Object must implement handle_event or be a callable!') + + @property + def cleanup(self): + return self._event_handler.cleanup diff --git a/lib/tfw/event_handlers/test_event_handler.py b/lib/tfw/event_handlers/test_event_handler.py new file mode 100644 index 0000000..2005f51 --- /dev/null +++ b/lib/tfw/event_handlers/test_event_handler.py @@ -0,0 +1,147 @@ +# pylint: disable=redefined-outer-name +from secrets import token_urlsafe +from random import randint + +import pytest + +from .event_handler_factory import EventHandlerFactoryBase +from .event_handler import EventHandler + + +class MockEventHandlerFactory(EventHandlerFactoryBase): + def _build_server_connector(self): + return MockServerConnector() + + +class MockServerConnector: + def __init__(self): + self.keys = [] + self._on_message = None + + def register_callback(self, callback): + self._on_message = callback + + def simulate_message(self, message): + self._on_message(message) + + def subscribe(self, keys): + self.keys.extend(keys) + + def unsubscribe(self, keys): + for key in keys: + self.keys.remove(key) + + def send_message(self, message, scope=None): + pass + + def close(self): + pass + + +class MockEventHandler: + def __init__(self): + self.cleaned_up = False + + def handle_event(self, message, server_connector): + pass + + def cleanup(self): + self.cleaned_up = True + + +@pytest.fixture +def test_msg(): + yield token_urlsafe(randint(16, 64)) + + +@pytest.fixture +def test_keys(): + yield [ + token_urlsafe(randint(2, 8)) + for _ in range(randint(16, 32)) + ] + + +def test_build_from_object(test_keys, test_msg): + mock_eh = MockEventHandler() + def test_handle_event(message, server_connector): + raise RuntimeError(message, server_connector.keys) + mock_eh.handle_event = test_handle_event + eh = MockEventHandlerFactory().build(mock_eh, keys=test_keys) + + with pytest.raises(RuntimeError) as err: + eh.server_connector.simulate_message(test_msg) + msg, keys = err.args + assert msg == test_msg + assert keys == test_keys + assert not mock_eh.cleaned_up + eh.stop() + assert mock_eh.cleaned_up + + +def test_build_from_object_with_keys(test_keys): + mock_eh = MockEventHandler() + mock_eh.keys = test_keys # pylint: disable=attribute-defined-outside-init + eh = MockEventHandlerFactory().build(mock_eh) + + assert not mock_eh.cleaned_up + EventHandler.stop_all_instances() + assert mock_eh.cleaned_up + assert eh.server_connector.keys == test_keys + + +def test_build_from_callable(test_keys, test_msg): + class SomeCallable: + def __init__(self): + self.message = None + self.cleaned_up = False + def __call__(self, message, server_connector): + self.message = message + def cleanup(self): + self.cleaned_up = True + + mock_eh = SomeCallable() + eh = MockEventHandlerFactory().build(mock_eh, keys=test_keys) + + assert eh.server_connector.keys == test_keys + assert not mock_eh.message + eh.server_connector.simulate_message(test_msg) + assert mock_eh.message == test_msg + assert not mock_eh.cleaned_up + eh.stop() + assert mock_eh.cleaned_up + + +def test_build_from_function(test_keys, test_msg): + def some_function(message, server_connector): + raise RuntimeError(message, server_connector.keys) + eh = MockEventHandlerFactory().build(some_function, keys=test_keys) + + assert eh.server_connector.keys == test_keys + with pytest.raises(RuntimeError) as err: + eh.server_connector.simulate_message(test_msg) + msg, keys = err.args + assert msg == test_msg + assert keys == test_keys + + +def test_build_from_lambda(test_keys, test_msg): + def assert_messages_equal(msg): + assert msg == test_msg + fun = lambda msg, sc: assert_messages_equal(msg) + eh = MockEventHandlerFactory().build(fun, keys=test_keys) + eh.server_connector.simulate_message(test_msg) + + +def test_build_raises_if_no_key(): + eh = MockEventHandler() + with pytest.raises(ValueError): + MockEventHandlerFactory().build(eh) + + def test_handle_event(*_): + pass + with pytest.raises(ValueError): + MockEventHandlerFactory().build(test_handle_event) + + with pytest.raises(ValueError): + MockEventHandlerFactory().build(lambda msg, sc: None) From f73c059429e8d8121340f0bcee4b0c2756055c40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 11 Jul 2019 18:52:15 +0200 Subject: [PATCH 079/174] Improve EventHandlerFactoryBase --- .../event_handlers/event_handler_factory.py | 33 ++++--- lib/tfw/event_handlers/test_event_handler.py | 93 +++++++++++++------ 2 files changed, 87 insertions(+), 39 deletions(-) diff --git a/lib/tfw/event_handlers/event_handler_factory.py b/lib/tfw/event_handlers/event_handler_factory.py index c3a6b18..dc03c46 100644 --- a/lib/tfw/event_handlers/event_handler_factory.py +++ b/lib/tfw/event_handlers/event_handler_factory.py @@ -4,25 +4,34 @@ from .event_handler import EventHandler class EventHandlerFactoryBase: - def build(self, event_handler, *, keys=None): - analyzer = EventHandlerAnalyzer(event_handler, keys) - event_handler = self._build_from_callable(analyzer) - event_handler.start() - return event_handler - - def _build_from_callable(self, analyzer): + def build(self, event_handler, *, keys=None, event_handler_type=EventHandler): + builder = EventHandlerBuilder(event_handler, keys, event_handler_type) server_connector = self._build_server_connector() - server_connector.subscribe(analyzer.keys) - event_handler = EventHandler(server_connector) - event_handler.handle_event = analyzer.handle_event + real_event_handler = builder.build(server_connector) + event_handler.server_connector = server_connector with suppress(AttributeError): - event_handler.cleanup = analyzer.cleanup - return event_handler + event_handler.start() + real_event_handler.start() + return real_event_handler def _build_server_connector(self): raise NotImplementedError() +class EventHandlerBuilder: + def __init__(self, event_handler, supplied_keys, event_handler_type): + self._analyzer = EventHandlerAnalyzer(event_handler, supplied_keys) + self._event_handler_type = event_handler_type + + def build(self, server_connector): + server_connector.subscribe(*self._analyzer.keys) + event_handler = self._event_handler_type(server_connector) + event_handler.handle_event = self._analyzer.handle_event + with suppress(AttributeError): + event_handler.cleanup = self._analyzer.cleanup + return event_handler + + class EventHandlerAnalyzer: def __init__(self, event_handler, supplied_keys): self._event_handler = event_handler diff --git a/lib/tfw/event_handlers/test_event_handler.py b/lib/tfw/event_handlers/test_event_handler.py index 2005f51..edde537 100644 --- a/lib/tfw/event_handlers/test_event_handler.py +++ b/lib/tfw/event_handlers/test_event_handler.py @@ -1,4 +1,4 @@ -# pylint: disable=redefined-outer-name +# pylint: disable=redefined-outer-name,attribute-defined-outside-init from secrets import token_urlsafe from random import randint @@ -18,16 +18,16 @@ class MockServerConnector: self.keys = [] self._on_message = None - def register_callback(self, callback): - self._on_message = callback - def simulate_message(self, message): self._on_message(message) - def subscribe(self, keys): + def register_callback(self, callback): + self._on_message = callback + + def subscribe(self, *keys): self.keys.extend(keys) - def unsubscribe(self, keys): + def unsubscribe(self, *keys): for key in keys: self.keys.remove(key) @@ -38,17 +38,31 @@ class MockServerConnector: pass -class MockEventHandler: +class MockEventHandlerStub: def __init__(self): + self.server_connector = None + self.last_message = None self.cleaned_up = False + self.started = False - def handle_event(self, message, server_connector): - pass + def start(self): + self.started = True def cleanup(self): self.cleaned_up = True +class MockEventHandler(MockEventHandlerStub): + # pylint: disable=unused-argument + def handle_event(self, message, server_connector): + self.last_message = message + + +class MockCallable(MockEventHandlerStub): + def __call__(self, message, server_connector): + self.last_message = message + + @pytest.fixture def test_msg(): yield token_urlsafe(randint(16, 64)) @@ -63,12 +77,16 @@ def test_keys(): def test_build_from_object(test_keys, test_msg): - mock_eh = MockEventHandler() - def test_handle_event(message, server_connector): + mock_eh = MockEventHandlerStub() + def handle_event(message, server_connector): raise RuntimeError(message, server_connector.keys) - mock_eh.handle_event = test_handle_event + mock_eh.handle_event = handle_event + + assert not mock_eh.started eh = MockEventHandlerFactory().build(mock_eh, keys=test_keys) + assert mock_eh.started + assert mock_eh.server_connector is eh.server_connector with pytest.raises(RuntimeError) as err: eh.server_connector.simulate_message(test_msg) msg, keys = err.args @@ -79,34 +97,52 @@ def test_build_from_object(test_keys, test_msg): assert mock_eh.cleaned_up -def test_build_from_object_with_keys(test_keys): +def test_build_from_object_with_keys(test_keys, test_msg): mock_eh = MockEventHandler() - mock_eh.keys = test_keys # pylint: disable=attribute-defined-outside-init + mock_eh.keys = test_keys + + assert not mock_eh.started eh = MockEventHandlerFactory().build(mock_eh) + assert mock_eh.server_connector.keys == test_keys + assert eh.server_connector is mock_eh.server_connector + assert mock_eh.started + assert not mock_eh.last_message + eh.server_connector.simulate_message(test_msg) + assert mock_eh.last_message == test_msg assert not mock_eh.cleaned_up EventHandler.stop_all_instances() assert mock_eh.cleaned_up - assert eh.server_connector.keys == test_keys + + +def test_build_from_simple_object(test_keys, test_msg): + class SimpleMockEventHandler: + # pylint: disable=no-self-use + def handle_event(self, message, server_connector): + raise RuntimeError(message, server_connector) + + mock_eh = SimpleMockEventHandler() + eh = MockEventHandlerFactory().build(mock_eh, keys=test_keys) + + with pytest.raises(RuntimeError) as err: + eh.server_connector.simulate_message(test_msg) + msg, keys = err.args + assert msg == test_msg + assert keys == test_keys def test_build_from_callable(test_keys, test_msg): - class SomeCallable: - def __init__(self): - self.message = None - self.cleaned_up = False - def __call__(self, message, server_connector): - self.message = message - def cleanup(self): - self.cleaned_up = True + mock_eh = MockCallable() - mock_eh = SomeCallable() + assert not mock_eh.started eh = MockEventHandlerFactory().build(mock_eh, keys=test_keys) + assert mock_eh.started + assert mock_eh.server_connector is eh.server_connector assert eh.server_connector.keys == test_keys - assert not mock_eh.message + assert not mock_eh.last_message eh.server_connector.simulate_message(test_msg) - assert mock_eh.message == test_msg + assert mock_eh.last_message == test_msg assert not mock_eh.cleaned_up eh.stop() assert mock_eh.cleaned_up @@ -133,7 +169,7 @@ def test_build_from_lambda(test_keys, test_msg): eh.server_connector.simulate_message(test_msg) -def test_build_raises_if_no_key(): +def test_build_raises_if_no_key(test_keys): eh = MockEventHandler() with pytest.raises(ValueError): MockEventHandlerFactory().build(eh) @@ -145,3 +181,6 @@ def test_build_raises_if_no_key(): with pytest.raises(ValueError): MockEventHandlerFactory().build(lambda msg, sc: None) + + eh.keys = test_keys + MockEventHandlerFactory().build(eh) From 42beafcf04e70e410b993fb00403c76d17ce398f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 12 Jul 2019 23:14:47 +0200 Subject: [PATCH 080/174] Update dependencies --- requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 610702d..82b480c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,7 @@ PyYAML>=5.0.0,<6.0.0 Jinja2>=2.0.0,<3.0.0 cryptography>=2.0.0,<3.0.0 python-dateutil>=2.0.0,<3.0.0 -SQLAlchemy==1.3.4 +SQLAlchemy>=1.0.0,<2.0.0 +pytest>=5.0.0,<6.0.0 +pylint>=2.0.0,<3.0.0 +rope>=0.0.0,<1.0.0 From de78f489300b54e8374b0e691e4fe4d41f195d83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 12 Jul 2019 23:25:16 +0200 Subject: [PATCH 081/174] Adjust the whole framework to event handler dependency inversion --- lib/tfw/builtins/__init__.py | 3 - .../directory_snapshotting_event_handler.py | 12 +- lib/tfw/builtins/event_handler.py | 9 -- lib/tfw/builtins/frontend_event_handler.py | 15 ++- lib/tfw/builtins/fsm_aware_event_handler.py | 20 --- .../builtins/fsm_managing_event_handler.py | 12 +- lib/tfw/builtins/ide_event_handler.py | 17 +-- .../builtins/log_monitoring_event_handler.py | 29 +++-- lib/tfw/builtins/pipe_io_event_handler.py | 24 ++-- .../process_managing_event_handler.py | 11 +- .../terminal_commands_event_handler.py | 11 +- lib/tfw/builtins/terminal_event_handler.py | 19 +-- lib/tfw/components/__init__.py | 1 - lib/tfw/components/fsm_updater.py | 2 +- lib/tfw/components/terminal_commands.py | 2 +- lib/tfw/event_handlers/__init__.py | 4 +- lib/tfw/event_handlers/event_handler_base.py | 117 ------------------ .../event_handlers/event_handler_factory.py | 10 +- .../fsm_aware.py | 27 ++-- .../event_handlers/fsm_aware_event_handler.py | 19 +++ lib/tfw/event_handlers/test_event_handler.py | 8 +- lib/tfw/main/__init__.py | 3 + lib/tfw/main/event_handler_factory.py | 8 ++ lib/tfw/main/signal_handling.py | 11 ++ .../tfw_connector.py} | 6 +- lib/tfw/networking/server_connector.py | 23 +++- 26 files changed, 169 insertions(+), 254 deletions(-) delete mode 100644 lib/tfw/builtins/event_handler.py delete mode 100644 lib/tfw/builtins/fsm_aware_event_handler.py delete mode 100644 lib/tfw/event_handlers/event_handler_base.py rename lib/tfw/{components => event_handlers}/fsm_aware.py (50%) create mode 100644 lib/tfw/event_handlers/fsm_aware_event_handler.py create mode 100644 lib/tfw/main/__init__.py create mode 100644 lib/tfw/main/event_handler_factory.py create mode 100644 lib/tfw/main/signal_handling.py rename lib/tfw/{builtins/tfw_server_connector.py => main/tfw_connector.py} (70%) diff --git a/lib/tfw/builtins/__init__.py b/lib/tfw/builtins/__init__.py index d276619..0343fe7 100644 --- a/lib/tfw/builtins/__init__.py +++ b/lib/tfw/builtins/__init__.py @@ -1,7 +1,5 @@ from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler -from .event_handler import EventHandler from .frontend_event_handler import FrontendEventHandler -from .fsm_aware_event_handler import FSMAwareEventHandler from .fsm_managing_event_handler import FSMManagingEventHandler from .ide_event_handler import IdeEventHandler from .log_monitoring_event_handler import LogMonitoringEventHandler @@ -10,4 +8,3 @@ from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHa from .process_managing_event_handler import ProcessManagingEventHandler from .terminal_commands_event_handler import TerminalCommandsEventHandler from .terminal_event_handler import TerminalEventHandler -from .tfw_server_connector import TFWServerUplinkConnector, TFWServerConnector diff --git a/lib/tfw/builtins/directory_snapshotting_event_handler.py b/lib/tfw/builtins/directory_snapshotting_event_handler.py index 25986eb..0029f9b 100644 --- a/lib/tfw/builtins/directory_snapshotting_event_handler.py +++ b/lib/tfw/builtins/directory_snapshotting_event_handler.py @@ -10,14 +10,14 @@ from tfw.components.snapshot_provider import SnapshotProvider from tfw.config import TFWENV from tfw.networking import Scope -from .event_handler import EventHandler LOG = logging.getLogger(__name__) -class DirectorySnapshottingEventHandler(EventHandler): - def __init__(self, key, directories, exclude_unix_patterns=None): - super().__init__(key, scope=Scope.WEBSOCKET) +class DirectorySnapshottingEventHandler: + keys = ['snapshot'] + + def __init__(self, directories, exclude_unix_patterns=None): self.snapshot_providers = {} self._exclude_unix_patterns = exclude_unix_patterns self.init_snapshot_providers(directories) @@ -46,11 +46,11 @@ class DirectorySnapshottingEventHandler(EventHandler): makedirs(git_dir, exist_ok=True) return git_dir - def handle_event(self, message): + def handle_event(self, message, server_connector): try: data = message['data'] message['data'] = self.command_handlers[data['command']](data) - self.send_message(message) + server_connector.send_message(message, scope=Scope.WEBSOCKET) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/lib/tfw/builtins/event_handler.py b/lib/tfw/builtins/event_handler.py deleted file mode 100644 index 9696fe4..0000000 --- a/lib/tfw/builtins/event_handler.py +++ /dev/null @@ -1,9 +0,0 @@ -from tfw.event_handlers import EventHandlerBase - -from .tfw_server_connector import TFWServerConnector - - -class EventHandler(EventHandlerBase): - # pylint: disable=abstract-method - def _build_server_connector(self): - return TFWServerConnector() diff --git a/lib/tfw/builtins/frontend_event_handler.py b/lib/tfw/builtins/frontend_event_handler.py index ba39c27..1fadc5b 100644 --- a/lib/tfw/builtins/frontend_event_handler.py +++ b/lib/tfw/builtins/frontend_event_handler.py @@ -1,16 +1,19 @@ from tfw.networking import Scope from tfw.components import FrontendMessageStorage -from .event_handler import EventHandler +class FrontendEventHandler: + keys = ['message', 'queueMessages', 'dashboard', 'console'] -class FrontendEventHandler(EventHandler): def __init__(self): - frontend_keys = ('message', 'queueMessages', 'dashboard', 'console') - self._frontend_message_storage = FrontendMessageStorage(frontend_keys) - super().__init__((*frontend_keys, 'recover'), scope=Scope.WEBSOCKET) + self.server_connector = None + self.keys = [*type(self).keys, 'recover'] + self._frontend_message_storage = FrontendMessageStorage(type(self).keys) - def handle_event(self, message): + def send_message(self, message): + self.server_connector.send_message(message, scope=Scope.WEBSOCKET) + + def handle_event(self, message, _): self._frontend_message_storage.save_message(message) if message['key'] == 'recover': self.recover_frontend() diff --git a/lib/tfw/builtins/fsm_aware_event_handler.py b/lib/tfw/builtins/fsm_aware_event_handler.py deleted file mode 100644 index d585651..0000000 --- a/lib/tfw/builtins/fsm_aware_event_handler.py +++ /dev/null @@ -1,20 +0,0 @@ -from tfw.components import FSMAware -from tfw.networking import Scope - -from .event_handler import EventHandler - - -class FSMAwareEventHandler(EventHandler, FSMAware): - # pylint: disable=abstract-method - """ - Abstract base class for EventHandlers which automatically - keep track of the state of the TFW FSM. - """ - def __init__(self, key, scope=Scope.ZMQ): - EventHandler.__init__(self, key, scope=scope) - FSMAware.__init__(self) - self.subscribe('fsm_update') - - def dispatch_handling(self, message): - if not self.refresh_on_fsm_update(message): - super().dispatch_handling(message) diff --git a/lib/tfw/builtins/fsm_managing_event_handler.py b/lib/tfw/builtins/fsm_managing_event_handler.py index ccbb7de..dbc568b 100644 --- a/lib/tfw/builtins/fsm_managing_event_handler.py +++ b/lib/tfw/builtins/fsm_managing_event_handler.py @@ -4,12 +4,12 @@ from tfw.crypto import KeyManager, sign_message, verify_message from tfw.networking import Scope from tfw.components import FSMUpdater -from .event_handler import EventHandler LOG = logging.getLogger(__name__) -class FSMManagingEventHandler(EventHandler): +class FSMManagingEventHandler: + keys = ['fsm'] """ EventHandler responsible for managing the state machine of the framework (TFW FSM). @@ -24,8 +24,7 @@ class FSMManagingEventHandler(EventHandler): An 'fsm_update' message is broadcasted after every successful command. """ - def __init__(self, key, fsm_type, require_signature=False): - super().__init__(key, scope=Scope.WEBSOCKET) + def __init__(self, fsm_type, require_signature=False): self.fsm = fsm_type() self._fsm_updater = FSMUpdater(self.fsm) self.auth_key = KeyManager().auth_key @@ -36,15 +35,14 @@ class FSMManagingEventHandler(EventHandler): 'update': self.handle_update } - def handle_event(self, message): + def handle_event(self, message, server_connector): try: message = self.command_handlers[message['data']['command']](message) if message: fsm_update_message = self._fsm_updater.fsm_update sign_message(self.auth_key, message) sign_message(self.auth_key, fsm_update_message) - self.server_connector.send_message(fsm_update_message, Scope.BROADCAST) - self.send_message(message) + server_connector.send_message(fsm_update_message, Scope.BROADCAST) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/lib/tfw/builtins/ide_event_handler.py b/lib/tfw/builtins/ide_event_handler.py index ab5a7eb..1ec4ebd 100644 --- a/lib/tfw/builtins/ide_event_handler.py +++ b/lib/tfw/builtins/ide_event_handler.py @@ -4,7 +4,6 @@ from tfw.networking import Scope from tfw.components import FileManager from tfw.components.inotify import InotifyObserver -from .event_handler import EventHandler LOG = logging.getLogger(__name__) @@ -32,7 +31,8 @@ BUILD_ARTIFACTS = ( ) -class IdeEventHandler(EventHandler): +class IdeEventHandler: + keys = ['ide'] # pylint: disable=too-many-arguments,anomalous-backslash-in-string """ Event handler implementing the backend of our browser based IDE. @@ -47,7 +47,7 @@ class IdeEventHandler(EventHandler): The API of each command is documented in their respective handler. """ - def __init__(self, key, directory, allowed_directories, selected_file=None, exclude=None): + def __init__(self, directory, allowed_directories, selected_file=None, exclude=None): """ :param key: the key this instance should listen to :param directory: working directory which the EventHandler should serve files from @@ -55,7 +55,7 @@ class IdeEventHandler(EventHandler): :param selected_file: file that is selected by default :param exclude: list of filenames that should not appear between files (for .o, .pyc, etc.) """ - super().__init__(key, scope=Scope.WEBSOCKET) + self.server_connector = None try: self.filemanager = FileManager( allowed_directories=allowed_directories, @@ -84,10 +84,13 @@ class IdeEventHandler(EventHandler): } def _reload_frontend(self, event): # pylint: disable=unused-argument - self.server_connector.send_message({ + self.send_message({ 'key': 'ide', 'data': {'command': 'reload'} - }, Scope.WEBSOCKET) + }) + + def send_message(self, message): + self.server_connector.send_message(message, scope=Scope.WEBSOCKET) def read(self, data): """ @@ -179,7 +182,7 @@ class IdeEventHandler(EventHandler): data['files'] = self.filemanager.files data['directory'] = self.filemanager.workdir - def handle_event(self, message): + def handle_event(self, message, _): try: data = message['data'] message['data'] = self.commands[data['command']](data) diff --git a/lib/tfw/builtins/log_monitoring_event_handler.py b/lib/tfw/builtins/log_monitoring_event_handler.py index eebade5..7e376fc 100644 --- a/lib/tfw/builtins/log_monitoring_event_handler.py +++ b/lib/tfw/builtins/log_monitoring_event_handler.py @@ -1,15 +1,14 @@ import logging from tfw.config import TFWENV -from tfw.networking import Scope from tfw.components import LogInotifyObserver -from .event_handler import EventHandler LOG = logging.getLogger(__name__) -class LogMonitoringEventHandler(EventHandler): +class LogMonitoringEventHandler: + keys = ['logmonitor'] """ Monitors the output of a supervisor process (stdout, stderr) and sends the results to the frontend. @@ -19,23 +18,27 @@ class LogMonitoringEventHandler(EventHandler): The API of each command is documented in their respective handler. """ - def __init__(self, key, process_name, log_tail=0): - super().__init__(key, scope=Scope.WEBSOCKET) + def __init__(self, process_name, log_tail=0): + self.server_connector = None self.process_name = process_name - self._monitor = LogInotifyObserver( - server_connector=self.server_connector, - supervisor_uri=TFWENV.SUPERVISOR_HTTP_URI, - process_name=process_name, - log_tail=log_tail - ) - self._monitor.start() + self._initial_log_tail = log_tail + self._monitor = None self.command_handlers = { 'process_name': self.handle_process_name, 'log_tail': self.handle_log_tail } - def handle_event(self, message): + def start(self): + self._monitor = LogInotifyObserver( + server_connector=self.server_connector, + supervisor_uri=TFWENV.SUPERVISOR_HTTP_URI, + process_name=self.process_name, + log_tail=self._initial_log_tail + ) + self._monitor.start() + + def handle_event(self, message, _): try: data = message['data'] self.command_handlers[data['command']](data) diff --git a/lib/tfw/builtins/pipe_io_event_handler.py b/lib/tfw/builtins/pipe_io_event_handler.py index 96bff90..0a8e02f 100644 --- a/lib/tfw/builtins/pipe_io_event_handler.py +++ b/lib/tfw/builtins/pipe_io_event_handler.py @@ -12,15 +12,16 @@ from contextlib import suppress from tfw.components.pipe_io_server import PipeIOServer, terminate_process_on_failure -from .event_handler import EventHandler LOG = logging.getLogger(__name__) DEFAULT_PERMISSIONS = 0o600 -class PipeIOEventHandlerBase(EventHandler): - def __init__(self, key, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS): - super().__init__(key) +class PipeIOEventHandlerBase: + keys = [''] + + def __init__(self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS): + self.server_connector = None self.pipe_io = CallbackPipeIOServer( in_pipe_path, out_pipe_path, @@ -50,25 +51,25 @@ class CallbackPipeIOServer(PipeIOServer): class PipeIOEventHandler(PipeIOEventHandlerBase): - def handle_event(self, message): + def handle_event(self, message, _): json_bytes = dumps(message).encode() self.pipe_io.send_message(json_bytes) def handle_pipe_event(self, message_bytes): json = loads(message_bytes) - self.send_message(json) + self.server_connector.send_message(json) class TransformerPipeIOEventHandler(PipeIOEventHandlerBase): # pylint: disable=too-many-arguments def __init__( - self, key, in_pipe_path, out_pipe_path, + self, in_pipe_path, out_pipe_path, transform_in_cmd, transform_out_cmd, permissions=DEFAULT_PERMISSIONS ): self._transform_in = partial(self._transform_message, transform_in_cmd) self._transform_out = partial(self._transform_message, transform_out_cmd) - super().__init__(key, in_pipe_path, out_pipe_path, permissions) + super().__init__(in_pipe_path, out_pipe_path, permissions) @staticmethod def _transform_message(transform_cmd, message): @@ -83,7 +84,7 @@ class TransformerPipeIOEventHandler(PipeIOEventHandlerBase): return proc.stdout raise ValueError(f'Transforming message {message} failed!') - def handle_event(self, message): + def handle_event(self, message, _): json_bytes = dumps(message).encode() transformed_bytes = self._transform_out(json_bytes) if transformed_bytes: @@ -93,13 +94,12 @@ class TransformerPipeIOEventHandler(PipeIOEventHandlerBase): transformed_bytes = self._transform_in(message_bytes) if transformed_bytes: json_message = loads(transformed_bytes) - self.send_message(json_message) + self.server_connector.send_message(json_message) class CommandEventHandler(PipeIOEventHandler): - def __init__(self, key, command, permissions=DEFAULT_PERMISSIONS): + def __init__(self, command, permissions=DEFAULT_PERMISSIONS): super().__init__( - key, self._generate_tempfilename(), self._generate_tempfilename(), permissions diff --git a/lib/tfw/builtins/process_managing_event_handler.py b/lib/tfw/builtins/process_managing_event_handler.py index 2a1f968..fe25ab6 100644 --- a/lib/tfw/builtins/process_managing_event_handler.py +++ b/lib/tfw/builtins/process_managing_event_handler.py @@ -5,12 +5,12 @@ from tfw.config import TFWENV from tfw.networking import Scope from tfw.components import ProcessManager, LogManager -from .event_handler import EventHandler LOG = logging.getLogger(__name__) -class ProcessManagingEventHandler(EventHandler, ProcessManager, LogManager): +class ProcessManagingEventHandler(ProcessManager, LogManager): + keys = ['processmanager'] """ Event handler that can manage processes managed by supervisor. @@ -23,8 +23,7 @@ class ProcessManagingEventHandler(EventHandler, ProcessManager, LogManager): Commands available: start, stop, restart, readlog (the names are as self-documenting as it gets) """ - def __init__(self, key, log_tail=0): - EventHandler.__init__(self, key, scope=Scope.WEBSOCKET) + def __init__(self, log_tail=0): ProcessManager.__init__(self, TFWENV.SUPERVISOR_HTTP_URI) LogManager.__init__(self, TFWENV.SUPERVISOR_HTTP_URI) self.log_tail = log_tail @@ -34,7 +33,7 @@ class ProcessManagingEventHandler(EventHandler, ProcessManager, LogManager): 'restart': self.restart_process } - def handle_event(self, message): + def handle_event(self, message, server_connector): try: data = message['data'] try: @@ -50,6 +49,6 @@ class ProcessManagingEventHandler(EventHandler, ProcessManager, LogManager): data['process_name'], self.log_tail ) - self.send_message(message) + server_connector.send_message(message, scope=Scope.WEBSOCKET) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/lib/tfw/builtins/terminal_commands_event_handler.py b/lib/tfw/builtins/terminal_commands_event_handler.py index 0385f0b..2bb7399 100644 --- a/lib/tfw/builtins/terminal_commands_event_handler.py +++ b/lib/tfw/builtins/terminal_commands_event_handler.py @@ -1,14 +1,9 @@ from tfw.components import TerminalCommands -from tfw.networking import Scope - -from .event_handler import EventHandler -class TerminalCommandsEventHandler(EventHandler, TerminalCommands): - def __init__(self, key, scope=Scope.ZMQ, bashrc=None): - EventHandler.__init__(self, key, scope) - TerminalCommands.__init__(self, bashrc) +class TerminalCommandsEventHandler(TerminalCommands): + keys = ['history.bash'] - def handle_event(self, message): + def handle_event(self, message, _): command = message['value'] self.callback(command) diff --git a/lib/tfw/builtins/terminal_event_handler.py b/lib/tfw/builtins/terminal_event_handler.py index 1f7bf5f..d02a058 100644 --- a/lib/tfw/builtins/terminal_event_handler.py +++ b/lib/tfw/builtins/terminal_event_handler.py @@ -1,16 +1,15 @@ import logging -from tfw.networking import Scope from tfw.components import BashMonitor, TerminadoMiniServer from tfw.config import TFWENV from tao.config import TAOENV -from .event_handler import EventHandler LOG = logging.getLogger(__name__) -class TerminalEventHandler(EventHandler): +class TerminalEventHandler: + keys = ['shell'] """ Event handler responsible for managing terminal sessions for frontend xterm sessions to connect to. You need to instanciate this in order for frontend @@ -20,13 +19,13 @@ class TerminalEventHandler(EventHandler): a command to be executed. The API of each command is documented in their respective handler. """ - def __init__(self, key): + def __init__(self): """ :param key: key this EventHandler listens to :param monitor: tfw.components.HistoryMonitor instance to read command history from """ - super().__init__(key, scope=Scope.WEBSOCKET) - self._historymonitor = BashMonitor(self.server_connector, TFWENV.HISTFILE) + self.server_connector = None + self._historymonitor = None bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash'] self.terminado_server = TerminadoMiniServer( @@ -41,18 +40,20 @@ class TerminalEventHandler(EventHandler): 'read': self.read } - self._historymonitor.start() self.terminado_server.listen() + def start(self): + self._historymonitor = BashMonitor(self.server_connector, TFWENV.HISTFILE) + self._historymonitor.start() + @property def historymonitor(self): return self._historymonitor - def handle_event(self, message): + def handle_event(self, message, _): try: data = message['data'] message['data'] = self.commands[data['command']](data) - self.send_message(message) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/lib/tfw/components/__init__.py b/lib/tfw/components/__init__.py index a1355d7..b44466f 100644 --- a/lib/tfw/components/__init__.py +++ b/lib/tfw/components/__init__.py @@ -1,6 +1,5 @@ from .commands_equal import CommandsEqual from .file_manager import FileManager -from .fsm_aware import FSMAware from .fsm_updater import FSMUpdater from .history_monitor import BashMonitor, GDBMonitor from .log_inotify_observer import LogInotifyObserver diff --git a/lib/tfw/components/fsm_updater.py b/lib/tfw/components/fsm_updater.py index 6bff16a..93f9e7a 100644 --- a/lib/tfw/components/fsm_updater.py +++ b/lib/tfw/components/fsm_updater.py @@ -6,7 +6,7 @@ class FSMUpdater: def fsm_update(self): return { 'key': 'fsm_update', - 'data': self.fsm_update_data + **self.fsm_update_data } @property diff --git a/lib/tfw/components/terminal_commands.py b/lib/tfw/components/terminal_commands.py index adba1b4..d705eb2 100644 --- a/lib/tfw/components/terminal_commands.py +++ b/lib/tfw/components/terminal_commands.py @@ -26,7 +26,7 @@ class TerminalCommands(ABC): You can also use this class to create new commands similarly. """ - def __init__(self, bashrc=None): + def __init__(self, bashrc): self._command_method_regex = r'^command_(.+)$' self.command_implemetations = self._build_command_to_implementation_dict() if bashrc is not None: diff --git a/lib/tfw/event_handlers/__init__.py b/lib/tfw/event_handlers/__init__.py index 7284ec8..8fc4177 100644 --- a/lib/tfw/event_handlers/__init__.py +++ b/lib/tfw/event_handlers/__init__.py @@ -1 +1,3 @@ -from .event_handler_base import EventHandlerBase +from .event_handler_factory import EventHandlerFactoryBase +from .event_handler import EventHandler +from .fsm_aware_event_handler import FSMAwareEventHandler diff --git a/lib/tfw/event_handlers/event_handler_base.py b/lib/tfw/event_handlers/event_handler_base.py deleted file mode 100644 index e56eef5..0000000 --- a/lib/tfw/event_handlers/event_handler_base.py +++ /dev/null @@ -1,117 +0,0 @@ -import logging -from abc import ABC, abstractmethod -from typing import Iterable - -from tfw.networking import Scope - -LOG = logging.getLogger(__name__) - - -class EventHandlerBase(ABC): - """ - Abstract base class for all Python based EventHandlers. Useful implementation template - for other languages. - - Derived classes must implement the handle_event() method - """ - _instances = set() - - def __init__(self, key, scope=Scope.ZMQ): - type(self)._instances.add(self) - self.server_connector = self._build_server_connector() - self.scope = scope - self.keys = [] - if isinstance(key, str): - self.keys.append(key) - elif isinstance(key, Iterable): - self.keys = list(key) - - self.subscribe(*self.keys) - self.server_connector.register_callback(self.event_handler_callback) - - @abstractmethod - def _build_server_connector(self): - raise NotImplementedError() - - def subscribe(self, *keys): - """ - Subscribe this EventHandler to receive events for given keys. - Note that you can subscribe to the same key several times in which - case you will need to unsubscribe multiple times in order to stop - receiving events. - - :param keys: list of keys to subscribe to - """ - for key in keys: - self.server_connector.subscribe(key) - self.keys.append(key) - - def event_handler_callback(self, message): - """ - Callback that is invoked when receiving a message. - Dispatches messages to handler methods and sends - a response back in case the handler returned something. - This is subscribed in __init__(). - """ - if self.check_key(message): - self.dispatch_handling(message) - - def check_key(self, message): - """ - Checks whether the message is intended for this - EventHandler. - - This is necessary because ZMQ handles PUB - SUB - connetions with pattern matching (e.g. someone - subscribed to 'fsm' will receive 'fsm_update' - messages as well. - """ - if '' in self.keys: - return True - return message['key'] in self.keys - - def dispatch_handling(self, message): - """ - Used to dispatch messages to their specific handlers. - - :param message: the message received - :returns: the message to send back - """ - self.handle_event(message) - - def handle_event(self, message): - """ - Abstract method that implements the handling of messages. - - :param message: the message received - :returns: the message to send back - """ - raise NotImplementedError() - - def send_message(self, message): - self.server_connector.send_message(message, self.scope) - - def unsubscribe(self, *keys): - """ - Unsubscribe this eventhandler from the given keys. - - :param keys: list of keys to unsubscribe from - """ - for key in keys: - self.server_connector.unsubscribe(key) - self.keys.remove(key) - - @classmethod - def stop_all_instances(cls): - for instance in cls._instances: - instance.stop() - - def stop(self): - self.server_connector.close() - self.cleanup() - - def cleanup(self): - """ - Perform cleanup actions such as releasing database - connections and stuff like that. - """ diff --git a/lib/tfw/event_handlers/event_handler_factory.py b/lib/tfw/event_handlers/event_handler_factory.py index dc03c46..52f6962 100644 --- a/lib/tfw/event_handlers/event_handler_factory.py +++ b/lib/tfw/event_handlers/event_handler_factory.py @@ -24,13 +24,21 @@ class EventHandlerBuilder: self._event_handler_type = event_handler_type def build(self, server_connector): - server_connector.subscribe(*self._analyzer.keys) event_handler = self._event_handler_type(server_connector) + server_connector.subscribe(*self._try_get_keys(event_handler)) event_handler.handle_event = self._analyzer.handle_event with suppress(AttributeError): event_handler.cleanup = self._analyzer.cleanup return event_handler + def _try_get_keys(self, event_handler): + try: + return self._analyzer.keys + except ValueError: + with suppress(AttributeError): + return event_handler.keys + raise + class EventHandlerAnalyzer: def __init__(self, event_handler, supplied_keys): diff --git a/lib/tfw/components/fsm_aware.py b/lib/tfw/event_handlers/fsm_aware.py similarity index 50% rename from lib/tfw/components/fsm_aware.py rename to lib/tfw/event_handlers/fsm_aware.py index 395a162..76d8d3f 100644 --- a/lib/tfw/components/fsm_aware.py +++ b/lib/tfw/event_handlers/fsm_aware.py @@ -6,6 +6,7 @@ LOG = logging.getLogger(__name__) class FSMAware: + keys = ['fsm_update'] """ Base class for stuff that has to be aware of the framework FSM. This is done by processing 'fsm_update' messages. @@ -16,27 +17,21 @@ class FSMAware: self.fsm_event_log = [] self._auth_key = KeyManager().auth_key - def refresh_on_fsm_update(self, message): - if message['key'] == 'fsm_update' and verify_message(self._auth_key, message): - self._handle_fsm_update(message) - return True - return False + def process_message(self, message): + if message['key'] == 'fsm_update': + if verify_message(self._auth_key, message): + self._handle_fsm_update(message) def _handle_fsm_update(self, message): try: - update_data = message['data'] - new_state = update_data['current_state'] + new_state = message['current_state'] if self.fsm_state != new_state: - self.handle_fsm_step(**update_data) + self.handle_fsm_step(message) self.fsm_state = new_state - self.fsm_in_accepted_state = update_data['in_accepted_state'] - self.fsm_event_log.append(update_data) + self.fsm_in_accepted_state = message['in_accepted_state'] + self.fsm_event_log.append(message) except KeyError: LOG.error('Invalid fsm_update message received!') - def handle_fsm_step(self, **kwargs): - """ - Called in case the TFW FSM has stepped. - - :param kwargs: fsm_update 'data' field - """ + def handle_fsm_step(self, message): + pass diff --git a/lib/tfw/event_handlers/fsm_aware_event_handler.py b/lib/tfw/event_handlers/fsm_aware_event_handler.py new file mode 100644 index 0000000..966d4d4 --- /dev/null +++ b/lib/tfw/event_handlers/fsm_aware_event_handler.py @@ -0,0 +1,19 @@ +from .event_handler import EventHandler +from .fsm_aware import FSMAware + + +class FSMAwareEventHandler(EventHandler, FSMAware): + # pylint: disable=abstract-method + """ + Abstract base class for EventHandlers which automatically + keep track of the state of the TFW FSM. + """ + def __init__(self, server_connector): + EventHandler.__init__(self, server_connector) + FSMAware.__init__(self) + + def _event_callback(self, message): + self.process_message(message) + + def handle_fsm_step(self, message): + self.handle_event(message, self.server_connector) diff --git a/lib/tfw/event_handlers/test_event_handler.py b/lib/tfw/event_handlers/test_event_handler.py index edde537..ae165f3 100644 --- a/lib/tfw/event_handlers/test_event_handler.py +++ b/lib/tfw/event_handlers/test_event_handler.py @@ -174,13 +174,17 @@ def test_build_raises_if_no_key(test_keys): with pytest.raises(ValueError): MockEventHandlerFactory().build(eh) - def test_handle_event(*_): + def handle_event(*_): pass with pytest.raises(ValueError): - MockEventHandlerFactory().build(test_handle_event) + MockEventHandlerFactory().build(handle_event) with pytest.raises(ValueError): MockEventHandlerFactory().build(lambda msg, sc: None) + WithKeysEventHandler = EventHandler + WithKeysEventHandler.keys = test_keys + MockEventHandlerFactory().build(eh, event_handler_type=WithKeysEventHandler) + eh.keys = test_keys MockEventHandlerFactory().build(eh) diff --git a/lib/tfw/main/__init__.py b/lib/tfw/main/__init__.py new file mode 100644 index 0000000..9e1c892 --- /dev/null +++ b/lib/tfw/main/__init__.py @@ -0,0 +1,3 @@ +from .tfw_connector import TFWUplinkConnector, TFWConnector +from .event_handler_factory import EventHandlerFactory +from .signal_handling import setup_signal_handlers diff --git a/lib/tfw/main/event_handler_factory.py b/lib/tfw/main/event_handler_factory.py new file mode 100644 index 0000000..271be8c --- /dev/null +++ b/lib/tfw/main/event_handler_factory.py @@ -0,0 +1,8 @@ +from tfw.event_handlers import EventHandlerFactoryBase + +from .tfw_connector import TFWConnector + + +class EventHandlerFactory(EventHandlerFactoryBase): + def _build_server_connector(self): + return TFWConnector() diff --git a/lib/tfw/main/signal_handling.py b/lib/tfw/main/signal_handling.py new file mode 100644 index 0000000..a8750ef --- /dev/null +++ b/lib/tfw/main/signal_handling.py @@ -0,0 +1,11 @@ +from signal import signal, SIGTERM, SIGINT + +from tfw.event_handlers import EventHandler + + +def setup_signal_handlers(): + def stop(*_): + EventHandler.stop_all_instances() + exit(0) + signal(SIGTERM, stop) + signal(SIGINT, stop) diff --git a/lib/tfw/builtins/tfw_server_connector.py b/lib/tfw/main/tfw_connector.py similarity index 70% rename from lib/tfw/builtins/tfw_server_connector.py rename to lib/tfw/main/tfw_connector.py index 8b8b7c3..2e56996 100644 --- a/lib/tfw/builtins/tfw_server_connector.py +++ b/lib/tfw/main/tfw_connector.py @@ -1,4 +1,4 @@ -from tfw.networking import ServerUplinkConnector, ServerConnector +from tfw.networking import ServerConnector, ServerUplinkConnector from tfw.config import TFWENV @@ -12,12 +12,12 @@ class ConnAddrMixin: return f'tcp://localhost:{TFWENV.PUB_PORT}' -class TFWServerUplinkConnector(ServerUplinkConnector, ConnAddrMixin): +class TFWUplinkConnector(ServerUplinkConnector, ConnAddrMixin): def __init__(self): super().__init__(self.uplink_conn_addr) -class TFWServerConnector(ServerConnector, ConnAddrMixin): +class TFWConnector(ServerConnector, ConnAddrMixin): def __init__(self): super().__init__( self.downlink_conn_addr, diff --git a/lib/tfw/networking/server_connector.py b/lib/tfw/networking/server_connector.py index da407ac..0ab1644 100644 --- a/lib/tfw/networking/server_connector.py +++ b/lib/tfw/networking/server_connector.py @@ -1,5 +1,4 @@ import logging -from functools import partial import zmq from zmq.eventloop.zmqstream import ZMQStream @@ -12,17 +11,31 @@ LOG = logging.getLogger(__name__) class ServerDownlinkConnector: def __init__(self, connect_addr): + self.keys = [] + self._on_recv_callback = None self._zmq_sub_socket = zmq.Context.instance().socket(zmq.SUB) self._zmq_sub_socket.setsockopt(zmq.RCVHWM, 0) self._zmq_sub_socket.connect(connect_addr) self._zmq_sub_stream = ZMQStream(self._zmq_sub_socket) - self.subscribe = partial(self._zmq_sub_socket.setsockopt_string, zmq.SUBSCRIBE) - self.unsubscribe = partial(self._zmq_sub_socket.setsockopt_string, zmq.UNSUBSCRIBE) + def subscribe(self, *keys): + for key in keys: + self._zmq_sub_socket.setsockopt_string(zmq.SUBSCRIBE, key) + self.keys.append(key) + + def unsubscribe(self, *keys): + for key in keys: + self._zmq_sub_socket.setsockopt_string(zmq.UNSUBSCRIBE, key) + self.keys.remove(key) def register_callback(self, callback): - callback = with_deserialize_tfw_msg(callback) - self._zmq_sub_stream.on_recv(callback) + self._on_recv_callback = callback + self._zmq_sub_stream.on_recv(with_deserialize_tfw_msg(self._on_recv)) + + def _on_recv(self, message): + key = message['key'] + if key in self.keys or '' in self.keys: + self._on_recv_callback(message) def close(self): self._zmq_sub_stream.close() From 026db9ea84c5f64478d958849f478ac518af6ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 12 Jul 2019 23:32:15 +0200 Subject: [PATCH 082/174] Avoid duplicate filename (EventHandlerFactory) --- lib/tfw/event_handlers/__init__.py | 2 +- .../{event_handler_factory.py => event_handler_factory_base.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename lib/tfw/event_handlers/{event_handler_factory.py => event_handler_factory_base.py} (100%) diff --git a/lib/tfw/event_handlers/__init__.py b/lib/tfw/event_handlers/__init__.py index 8fc4177..2a33065 100644 --- a/lib/tfw/event_handlers/__init__.py +++ b/lib/tfw/event_handlers/__init__.py @@ -1,3 +1,3 @@ -from .event_handler_factory import EventHandlerFactoryBase +from .event_handler_factory_base import EventHandlerFactoryBase from .event_handler import EventHandler from .fsm_aware_event_handler import FSMAwareEventHandler diff --git a/lib/tfw/event_handlers/event_handler_factory.py b/lib/tfw/event_handlers/event_handler_factory_base.py similarity index 100% rename from lib/tfw/event_handlers/event_handler_factory.py rename to lib/tfw/event_handlers/event_handler_factory_base.py From adf2672cd425d2f1f5f72caaf7cebfabd5b4bfc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 12 Jul 2019 23:38:18 +0200 Subject: [PATCH 083/174] Add forgotten dateutil dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 82b480c..3c34b45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ Jinja2>=2.0.0,<3.0.0 cryptography>=2.0.0,<3.0.0 python-dateutil>=2.0.0,<3.0.0 SQLAlchemy>=1.0.0,<2.0.0 +python-dateutil>=2.0.0,<3.0.0 pytest>=5.0.0,<6.0.0 pylint>=2.0.0,<3.0.0 rope>=0.0.0,<1.0.0 From 45a9a753b25c5f21021e7972c8dd10add18307ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 15 Jul 2019 11:09:43 +0200 Subject: [PATCH 084/174] Fix test_event_handler --- lib/tfw/event_handlers/test_event_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tfw/event_handlers/test_event_handler.py b/lib/tfw/event_handlers/test_event_handler.py index ae165f3..049bb78 100644 --- a/lib/tfw/event_handlers/test_event_handler.py +++ b/lib/tfw/event_handlers/test_event_handler.py @@ -4,7 +4,7 @@ from random import randint import pytest -from .event_handler_factory import EventHandlerFactoryBase +from .event_handler_factory_base import EventHandlerFactoryBase from .event_handler import EventHandler @@ -71,7 +71,7 @@ def test_msg(): @pytest.fixture def test_keys(): yield [ - token_urlsafe(randint(2, 8)) + token_urlsafe(randint(2, 8)) for _ in range(randint(16, 32)) ] From 9725f805edf0803be50d03ba4e9f72e07a2cf111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 15 Jul 2019 11:09:57 +0200 Subject: [PATCH 085/174] Silence unjust pylint warnings --- lib/tfw/server/zmq_websocket_router.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tfw/server/zmq_websocket_router.py b/lib/tfw/server/zmq_websocket_router.py index cb19c33..b7adf16 100644 --- a/lib/tfw/server/zmq_websocket_router.py +++ b/lib/tfw/server/zmq_websocket_router.py @@ -9,10 +9,10 @@ LOG = logging.getLogger(__name__) class ZMQWebSocketRouter(WebSocketHandler): - # pylint: disable=abstract-method + # pylint: disable=abstract-method,attribute-defined-outside-init instances = set() - def initialize(self, **kwargs): # pylint: disable=arguments-differ + def initialize(self, **kwargs): self.event_handler_connector = kwargs['event_handler_connector'] self.tfw_router = TFWRouter(self.send_to_zmq, self.send_to_websockets) From ae69a094c7459ec9963d45aa19592e30287accff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 23 Jul 2019 11:47:59 +0200 Subject: [PATCH 086/174] Improve names in EventHandlerFactoryBase to improve readability --- .../event_handler_factory_base.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/tfw/event_handlers/event_handler_factory_base.py b/lib/tfw/event_handlers/event_handler_factory_base.py index 52f6962..4c4adcd 100644 --- a/lib/tfw/event_handlers/event_handler_factory_base.py +++ b/lib/tfw/event_handlers/event_handler_factory_base.py @@ -4,15 +4,15 @@ from .event_handler import EventHandler class EventHandlerFactoryBase: - def build(self, event_handler, *, keys=None, event_handler_type=EventHandler): - builder = EventHandlerBuilder(event_handler, keys, event_handler_type) + def build(self, handler_stub, *, keys=None, event_handler_type=EventHandler): + builder = EventHandlerBuilder(handler_stub, keys, event_handler_type) server_connector = self._build_server_connector() - real_event_handler = builder.build(server_connector) - event_handler.server_connector = server_connector + event_handler = builder.build(server_connector) + handler_stub.server_connector = server_connector with suppress(AttributeError): - event_handler.start() - real_event_handler.start() - return real_event_handler + handler_stub.start() + event_handler.start() + return event_handler def _build_server_connector(self): raise NotImplementedError() @@ -20,7 +20,7 @@ class EventHandlerFactoryBase: class EventHandlerBuilder: def __init__(self, event_handler, supplied_keys, event_handler_type): - self._analyzer = EventHandlerAnalyzer(event_handler, supplied_keys) + self._analyzer = HandlerStubAnalyzer(event_handler, supplied_keys) self._event_handler_type = event_handler_type def build(self, server_connector): @@ -40,7 +40,7 @@ class EventHandlerBuilder: raise -class EventHandlerAnalyzer: +class HandlerStubAnalyzer: def __init__(self, event_handler, supplied_keys): self._event_handler = event_handler self._supplied_keys = supplied_keys From c6e01d294df346e16edd6029a0d78de8fdff76eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 23 Jul 2019 15:32:50 +0200 Subject: [PATCH 087/174] Inject TFWENV dependencies to builtins --- .../directory_snapshotting_event_handler.py | 9 ++++----- lib/tfw/builtins/fsm_managing_event_handler.py | 2 +- lib/tfw/builtins/ide_event_handler.py | 2 +- lib/tfw/builtins/log_monitoring_event_handler.py | 6 +++--- lib/tfw/builtins/process_managing_event_handler.py | 7 +++---- lib/tfw/builtins/terminal_event_handler.py | 13 ++++++------- 6 files changed, 18 insertions(+), 21 deletions(-) diff --git a/lib/tfw/builtins/directory_snapshotting_event_handler.py b/lib/tfw/builtins/directory_snapshotting_event_handler.py index 0029f9b..91aff92 100644 --- a/lib/tfw/builtins/directory_snapshotting_event_handler.py +++ b/lib/tfw/builtins/directory_snapshotting_event_handler.py @@ -7,7 +7,6 @@ from datetime import datetime from dateutil import parser as dateparser from tfw.components.snapshot_provider import SnapshotProvider -from tfw.config import TFWENV from tfw.networking import Scope @@ -17,7 +16,8 @@ LOG = logging.getLogger(__name__) class DirectorySnapshottingEventHandler: keys = ['snapshot'] - def __init__(self, directories, exclude_unix_patterns=None): + def __init__(self, *, directories, snapshots_dir, exclude_unix_patterns=None): + self._snapshots_dir = snapshots_dir self.snapshot_providers = {} self._exclude_unix_patterns = exclude_unix_patterns self.init_snapshot_providers(directories) @@ -37,10 +37,9 @@ class DirectorySnapshottingEventHandler: self._exclude_unix_patterns ) - @staticmethod - def init_git_dir(index, directory): + def init_git_dir(self, index, directory): git_dir = joinpath( - TFWENV.SNAPSHOTS_DIR, + self._snapshots_dir, f'{basename(directory)}-{index}' ) makedirs(git_dir, exist_ok=True) diff --git a/lib/tfw/builtins/fsm_managing_event_handler.py b/lib/tfw/builtins/fsm_managing_event_handler.py index dbc568b..2bf7fa2 100644 --- a/lib/tfw/builtins/fsm_managing_event_handler.py +++ b/lib/tfw/builtins/fsm_managing_event_handler.py @@ -24,7 +24,7 @@ class FSMManagingEventHandler: An 'fsm_update' message is broadcasted after every successful command. """ - def __init__(self, fsm_type, require_signature=False): + def __init__(self, *, fsm_type, require_signature=False): self.fsm = fsm_type() self._fsm_updater = FSMUpdater(self.fsm) self.auth_key = KeyManager().auth_key diff --git a/lib/tfw/builtins/ide_event_handler.py b/lib/tfw/builtins/ide_event_handler.py index 1ec4ebd..6cff1f4 100644 --- a/lib/tfw/builtins/ide_event_handler.py +++ b/lib/tfw/builtins/ide_event_handler.py @@ -47,7 +47,7 @@ class IdeEventHandler: The API of each command is documented in their respective handler. """ - def __init__(self, directory, allowed_directories, selected_file=None, exclude=None): + def __init__(self, *, directory, allowed_directories, selected_file=None, exclude=None): """ :param key: the key this instance should listen to :param directory: working directory which the EventHandler should serve files from diff --git a/lib/tfw/builtins/log_monitoring_event_handler.py b/lib/tfw/builtins/log_monitoring_event_handler.py index 7e376fc..54927eb 100644 --- a/lib/tfw/builtins/log_monitoring_event_handler.py +++ b/lib/tfw/builtins/log_monitoring_event_handler.py @@ -1,6 +1,5 @@ import logging -from tfw.config import TFWENV from tfw.components import LogInotifyObserver @@ -18,9 +17,10 @@ class LogMonitoringEventHandler: The API of each command is documented in their respective handler. """ - def __init__(self, process_name, log_tail=0): + def __init__(self, *, process_name, supervisor_uri, log_tail=0): self.server_connector = None self.process_name = process_name + self._supervisor_uri = supervisor_uri self._initial_log_tail = log_tail self._monitor = None @@ -32,7 +32,7 @@ class LogMonitoringEventHandler: def start(self): self._monitor = LogInotifyObserver( server_connector=self.server_connector, - supervisor_uri=TFWENV.SUPERVISOR_HTTP_URI, + supervisor_uri=self._supervisor_uri, process_name=self.process_name, log_tail=self._initial_log_tail ) diff --git a/lib/tfw/builtins/process_managing_event_handler.py b/lib/tfw/builtins/process_managing_event_handler.py index fe25ab6..41eb114 100644 --- a/lib/tfw/builtins/process_managing_event_handler.py +++ b/lib/tfw/builtins/process_managing_event_handler.py @@ -1,7 +1,6 @@ import logging from xmlrpc.client import Fault as SupervisorFault -from tfw.config import TFWENV from tfw.networking import Scope from tfw.components import ProcessManager, LogManager @@ -23,9 +22,9 @@ class ProcessManagingEventHandler(ProcessManager, LogManager): Commands available: start, stop, restart, readlog (the names are as self-documenting as it gets) """ - def __init__(self, log_tail=0): - ProcessManager.__init__(self, TFWENV.SUPERVISOR_HTTP_URI) - LogManager.__init__(self, TFWENV.SUPERVISOR_HTTP_URI) + def __init__(self, *, supervisor_uri, log_tail=0): + ProcessManager.__init__(self, supervisor_uri) + LogManager.__init__(self, supervisor_uri) self.log_tail = log_tail self.commands = { 'start': self.start_process, diff --git a/lib/tfw/builtins/terminal_event_handler.py b/lib/tfw/builtins/terminal_event_handler.py index d02a058..6cee755 100644 --- a/lib/tfw/builtins/terminal_event_handler.py +++ b/lib/tfw/builtins/terminal_event_handler.py @@ -1,8 +1,6 @@ import logging from tfw.components import BashMonitor, TerminadoMiniServer -from tfw.config import TFWENV -from tao.config import TAOENV LOG = logging.getLogger(__name__) @@ -19,19 +17,20 @@ class TerminalEventHandler: a command to be executed. The API of each command is documented in their respective handler. """ - def __init__(self): + def __init__(self, *, port, user, workind_directory, histfile): """ :param key: key this EventHandler listens to :param monitor: tfw.components.HistoryMonitor instance to read command history from """ self.server_connector = None + self._histfile = histfile self._historymonitor = None - bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash'] + bash_as_user_cmd = ['sudo', '-u', user, 'bash'] self.terminado_server = TerminadoMiniServer( '/terminal', - TFWENV.TERMINADO_PORT, - TFWENV.TERMINADO_WD, + port, + workind_directory, bash_as_user_cmd ) @@ -43,7 +42,7 @@ class TerminalEventHandler: self.terminado_server.listen() def start(self): - self._historymonitor = BashMonitor(self.server_connector, TFWENV.HISTFILE) + self._historymonitor = BashMonitor(self.server_connector, self._histfile) self._historymonitor.start() @property From a23224acedab87697d9bd1fef5a8213001baba71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 24 Jul 2019 15:17:16 +0200 Subject: [PATCH 088/174] Rework whole package structure (improved dependency handling) --- lib/envvars/__init__.py | 2 +- lib/tfw/builtins/__init__.py | 10 ---------- lib/tfw/components/__init__.py | 11 ----------- lib/tfw/components/frontend/__init__.py | 2 ++ .../frontend/frontend_handler.py} | 7 ++++--- lib/tfw/components/{ => frontend}/message_sender.py | 0 lib/tfw/components/{ => frontend}/message_storage.py | 0 lib/tfw/components/fsm/__init__.py | 1 + .../fsm/fsm_handler.py} | 9 +++++---- lib/tfw/components/{ => fsm}/fsm_updater.py | 0 lib/tfw/components/ide/__init__.py | 1 + lib/tfw/components/{ => ide}/file_manager/__init__.py | 0 .../components/{ => ide}/file_manager/file_manager.py | 0 .../{ => ide}/file_manager/test_file_manager.py | 0 .../ide/ide_handler.py} | 9 +++++---- lib/tfw/components/pipe_io/__init__.py | 1 + .../pipe_io/pipe_io_handler.py} | 10 +++++----- .../{ => pipe_io}/pipe_io_server/__init__.py | 0 .../components/{ => pipe_io}/pipe_io_server/deque.py | 0 .../components/{ => pipe_io}/pipe_io_server/pipe.py | 0 .../{ => pipe_io}/pipe_io_server/pipe_io_server.py | 0 .../pipe_io_server/pipe_reader_thread.py | 0 .../pipe_io_server/pipe_writer_thread.py | 0 .../pipe_io_server/terminate_process_on_failure.py | 0 lib/tfw/components/process_management/__init__.py | 2 ++ .../{ => process_management}/log_inotify_observer.py | 10 +++++----- .../process_management/process_handler.py} | 9 +++++---- .../process_management/process_log_handler.py} | 4 ++-- .../components/{ => process_management}/supervisor.py | 2 +- lib/tfw/components/snapshots/__init__.py | 1 + .../snapshots/snapshot_handler.py} | 7 ++++--- .../components/{ => snapshots}/snapshot_provider.py | 0 lib/tfw/components/terminal/__init__.py | 3 +++ lib/tfw/components/{ => terminal}/commands_equal.py | 2 +- lib/tfw/components/{ => terminal}/history_monitor.py | 2 +- .../{ => terminal}/terminado_mini_server.py | 0 .../components/{ => terminal}/terminal_commands.py | 0 .../terminal/terminal_commands_handler.py} | 4 ++-- .../terminal/terminal_handler.py} | 5 +++-- lib/tfw/event_handlers.py | 2 ++ lib/tfw/fsm/fsm_base.py | 2 +- lib/tfw/fsm/linear_fsm.py | 2 +- lib/tfw/fsm/yaml_fsm.py | 2 +- lib/tfw/{decorators => internals}/__init__.py | 0 lib/tfw/{mixins => internals}/callback_mixin.py | 2 +- lib/tfw/{ => internals}/crypto.py | 4 ++-- .../event_handling}/__init__.py | 0 .../event_handling}/event_handler.py | 0 .../event_handling}/event_handler_factory_base.py | 0 .../event_handling}/fsm_aware.py | 2 +- .../event_handling}/fsm_aware_event_handler.py | 0 .../event_handling}/test_event_handler.py | 0 lib/tfw/{components => internals}/inotify/__init__.py | 0 lib/tfw/{components => internals}/inotify/inotify.py | 0 .../{components => internals}/inotify/test_inotify.py | 0 .../lazy_property.py => internals/lazy.py} | 0 lib/tfw/{ => internals}/networking/__init__.py | 0 .../networking/event_handler_connector.py | 0 lib/tfw/{ => internals}/networking/scope.py | 0 lib/tfw/{ => internals}/networking/serialization.py | 0 .../{ => internals}/networking/server_connector.py | 0 lib/tfw/internals/server/__init__.py | 1 + .../{ => internals}/server/zmq_websocket_router.py | 2 +- lib/tfw/logging.py | 2 +- lib/tfw/main/__init__.py | 1 + lib/tfw/main/event_handler_factory.py | 2 +- lib/tfw/main/signal_handling.py | 2 +- lib/tfw/main/tfw_connector.py | 2 +- lib/tfw/{server => main}/tfw_server.py | 4 ++-- lib/tfw/mixins/__init__.py | 0 lib/tfw/server/__init__.py | 1 - supervisor/tfw_server.py | 2 +- 72 files changed, 74 insertions(+), 75 deletions(-) delete mode 100644 lib/tfw/builtins/__init__.py create mode 100644 lib/tfw/components/frontend/__init__.py rename lib/tfw/{builtins/frontend_event_handler.py => components/frontend/frontend_handler.py} (85%) rename lib/tfw/components/{ => frontend}/message_sender.py (100%) rename lib/tfw/components/{ => frontend}/message_storage.py (100%) create mode 100644 lib/tfw/components/fsm/__init__.py rename lib/tfw/{builtins/fsm_managing_event_handler.py => components/fsm/fsm_handler.py} (92%) rename lib/tfw/components/{ => fsm}/fsm_updater.py (100%) create mode 100644 lib/tfw/components/ide/__init__.py rename lib/tfw/components/{ => ide}/file_manager/__init__.py (100%) rename lib/tfw/components/{ => ide}/file_manager/file_manager.py (100%) rename lib/tfw/components/{ => ide}/file_manager/test_file_manager.py (100%) rename lib/tfw/{builtins/ide_event_handler.py => components/ide/ide_handler.py} (97%) create mode 100644 lib/tfw/components/pipe_io/__init__.py rename lib/tfw/{builtins/pipe_io_event_handler.py => components/pipe_io/pipe_io_handler.py} (94%) rename lib/tfw/components/{ => pipe_io}/pipe_io_server/__init__.py (100%) rename lib/tfw/components/{ => pipe_io}/pipe_io_server/deque.py (100%) rename lib/tfw/components/{ => pipe_io}/pipe_io_server/pipe.py (100%) rename lib/tfw/components/{ => pipe_io}/pipe_io_server/pipe_io_server.py (100%) rename lib/tfw/components/{ => pipe_io}/pipe_io_server/pipe_reader_thread.py (100%) rename lib/tfw/components/{ => pipe_io}/pipe_io_server/pipe_writer_thread.py (100%) rename lib/tfw/components/{ => pipe_io}/pipe_io_server/terminate_process_on_failure.py (100%) create mode 100644 lib/tfw/components/process_management/__init__.py rename lib/tfw/components/{ => process_management}/log_inotify_observer.py (84%) rename lib/tfw/{builtins/process_managing_event_handler.py => components/process_management/process_handler.py} (88%) rename lib/tfw/{builtins/log_monitoring_event_handler.py => components/process_management/process_log_handler.py} (96%) rename lib/tfw/components/{ => process_management}/supervisor.py (96%) create mode 100644 lib/tfw/components/snapshots/__init__.py rename lib/tfw/{builtins/directory_snapshotting_event_handler.py => components/snapshots/snapshot_handler.py} (95%) rename lib/tfw/components/{ => snapshots}/snapshot_provider.py (100%) create mode 100644 lib/tfw/components/terminal/__init__.py rename lib/tfw/components/{ => terminal}/commands_equal.py (98%) rename lib/tfw/components/{ => terminal}/history_monitor.py (98%) rename lib/tfw/components/{ => terminal}/terminado_mini_server.py (100%) rename lib/tfw/components/{ => terminal}/terminal_commands.py (100%) rename lib/tfw/{builtins/terminal_commands_event_handler.py => components/terminal/terminal_commands_handler.py} (58%) rename lib/tfw/{builtins/terminal_event_handler.py => components/terminal/terminal_handler.py} (95%) create mode 100644 lib/tfw/event_handlers.py rename lib/tfw/{decorators => internals}/__init__.py (100%) rename lib/tfw/{mixins => internals}/callback_mixin.py (95%) rename lib/tfw/{ => internals}/crypto.py (96%) rename lib/tfw/{event_handlers => internals/event_handling}/__init__.py (100%) rename lib/tfw/{event_handlers => internals/event_handling}/event_handler.py (100%) rename lib/tfw/{event_handlers => internals/event_handling}/event_handler_factory_base.py (100%) rename lib/tfw/{event_handlers => internals/event_handling}/fsm_aware.py (94%) rename lib/tfw/{event_handlers => internals/event_handling}/fsm_aware_event_handler.py (100%) rename lib/tfw/{event_handlers => internals/event_handling}/test_event_handler.py (100%) rename lib/tfw/{components => internals}/inotify/__init__.py (100%) rename lib/tfw/{components => internals}/inotify/inotify.py (100%) rename lib/tfw/{components => internals}/inotify/test_inotify.py (100%) rename lib/tfw/{decorators/lazy_property.py => internals/lazy.py} (100%) rename lib/tfw/{ => internals}/networking/__init__.py (100%) rename lib/tfw/{ => internals}/networking/event_handler_connector.py (100%) rename lib/tfw/{ => internals}/networking/scope.py (100%) rename lib/tfw/{ => internals}/networking/serialization.py (100%) rename lib/tfw/{ => internals}/networking/server_connector.py (100%) create mode 100644 lib/tfw/internals/server/__init__.py rename lib/tfw/{ => internals}/server/zmq_websocket_router.py (97%) rename lib/tfw/{server => main}/tfw_server.py (88%) delete mode 100644 lib/tfw/mixins/__init__.py delete mode 100644 lib/tfw/server/__init__.py diff --git a/lib/envvars/__init__.py b/lib/envvars/__init__.py index a96ed9f..db9a3df 100644 --- a/lib/envvars/__init__.py +++ b/lib/envvars/__init__.py @@ -1,7 +1,7 @@ from collections import namedtuple from os import environ -from tfw.decorators.lazy_property import lazy_property +from tfw.internals.lazy import lazy_property class LazyEnvironment: diff --git a/lib/tfw/builtins/__init__.py b/lib/tfw/builtins/__init__.py deleted file mode 100644 index 0343fe7..0000000 --- a/lib/tfw/builtins/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler -from .frontend_event_handler import FrontendEventHandler -from .fsm_managing_event_handler import FSMManagingEventHandler -from .ide_event_handler import IdeEventHandler -from .log_monitoring_event_handler import LogMonitoringEventHandler -from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler -from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler -from .process_managing_event_handler import ProcessManagingEventHandler -from .terminal_commands_event_handler import TerminalCommandsEventHandler -from .terminal_event_handler import TerminalEventHandler diff --git a/lib/tfw/components/__init__.py b/lib/tfw/components/__init__.py index b44466f..e69de29 100644 --- a/lib/tfw/components/__init__.py +++ b/lib/tfw/components/__init__.py @@ -1,11 +0,0 @@ -from .commands_equal import CommandsEqual -from .file_manager import FileManager -from .fsm_updater import FSMUpdater -from .history_monitor import BashMonitor, GDBMonitor -from .log_inotify_observer import LogInotifyObserver -from .message_sender import MessageSender -from .message_storage import FrontendMessageStorage -from .snapshot_provider import SnapshotProvider -from .supervisor import ProcessManager, LogManager -from .terminado_mini_server import TerminadoMiniServer -from .terminal_commands import TerminalCommands diff --git a/lib/tfw/components/frontend/__init__.py b/lib/tfw/components/frontend/__init__.py new file mode 100644 index 0000000..3e0b871 --- /dev/null +++ b/lib/tfw/components/frontend/__init__.py @@ -0,0 +1,2 @@ +from .frontend_handler import FrontendHandler +from .message_sender import MessageSender diff --git a/lib/tfw/builtins/frontend_event_handler.py b/lib/tfw/components/frontend/frontend_handler.py similarity index 85% rename from lib/tfw/builtins/frontend_event_handler.py rename to lib/tfw/components/frontend/frontend_handler.py index 1fadc5b..b5e7c8b 100644 --- a/lib/tfw/builtins/frontend_event_handler.py +++ b/lib/tfw/components/frontend/frontend_handler.py @@ -1,8 +1,9 @@ -from tfw.networking import Scope -from tfw.components import FrontendMessageStorage +from tfw.internals.networking import Scope + +from .message_storage import FrontendMessageStorage -class FrontendEventHandler: +class FrontendHandler: keys = ['message', 'queueMessages', 'dashboard', 'console'] def __init__(self): diff --git a/lib/tfw/components/message_sender.py b/lib/tfw/components/frontend/message_sender.py similarity index 100% rename from lib/tfw/components/message_sender.py rename to lib/tfw/components/frontend/message_sender.py diff --git a/lib/tfw/components/message_storage.py b/lib/tfw/components/frontend/message_storage.py similarity index 100% rename from lib/tfw/components/message_storage.py rename to lib/tfw/components/frontend/message_storage.py diff --git a/lib/tfw/components/fsm/__init__.py b/lib/tfw/components/fsm/__init__.py new file mode 100644 index 0000000..5e8f625 --- /dev/null +++ b/lib/tfw/components/fsm/__init__.py @@ -0,0 +1 @@ +from .fsm_handler import FSMHandler diff --git a/lib/tfw/builtins/fsm_managing_event_handler.py b/lib/tfw/components/fsm/fsm_handler.py similarity index 92% rename from lib/tfw/builtins/fsm_managing_event_handler.py rename to lib/tfw/components/fsm/fsm_handler.py index 2bf7fa2..2dc163e 100644 --- a/lib/tfw/builtins/fsm_managing_event_handler.py +++ b/lib/tfw/components/fsm/fsm_handler.py @@ -1,14 +1,15 @@ import logging -from tfw.crypto import KeyManager, sign_message, verify_message -from tfw.networking import Scope -from tfw.components import FSMUpdater +from tfw.internals.crypto import KeyManager, sign_message, verify_message +from tfw.internals.networking import Scope + +from .fsm_updater import FSMUpdater LOG = logging.getLogger(__name__) -class FSMManagingEventHandler: +class FSMHandler: keys = ['fsm'] """ EventHandler responsible for managing the state machine of diff --git a/lib/tfw/components/fsm_updater.py b/lib/tfw/components/fsm/fsm_updater.py similarity index 100% rename from lib/tfw/components/fsm_updater.py rename to lib/tfw/components/fsm/fsm_updater.py diff --git a/lib/tfw/components/ide/__init__.py b/lib/tfw/components/ide/__init__.py new file mode 100644 index 0000000..2d4ae3e --- /dev/null +++ b/lib/tfw/components/ide/__init__.py @@ -0,0 +1 @@ +from .ide_handler import IdeHandler diff --git a/lib/tfw/components/file_manager/__init__.py b/lib/tfw/components/ide/file_manager/__init__.py similarity index 100% rename from lib/tfw/components/file_manager/__init__.py rename to lib/tfw/components/ide/file_manager/__init__.py diff --git a/lib/tfw/components/file_manager/file_manager.py b/lib/tfw/components/ide/file_manager/file_manager.py similarity index 100% rename from lib/tfw/components/file_manager/file_manager.py rename to lib/tfw/components/ide/file_manager/file_manager.py diff --git a/lib/tfw/components/file_manager/test_file_manager.py b/lib/tfw/components/ide/file_manager/test_file_manager.py similarity index 100% rename from lib/tfw/components/file_manager/test_file_manager.py rename to lib/tfw/components/ide/file_manager/test_file_manager.py diff --git a/lib/tfw/builtins/ide_event_handler.py b/lib/tfw/components/ide/ide_handler.py similarity index 97% rename from lib/tfw/builtins/ide_event_handler.py rename to lib/tfw/components/ide/ide_handler.py index 6cff1f4..225b80c 100644 --- a/lib/tfw/builtins/ide_event_handler.py +++ b/lib/tfw/components/ide/ide_handler.py @@ -1,8 +1,9 @@ import logging -from tfw.networking import Scope -from tfw.components import FileManager -from tfw.components.inotify import InotifyObserver +from tfw.internals.networking import Scope +from tfw.internals.inotify import InotifyObserver + +from .file_manager import FileManager LOG = logging.getLogger(__name__) @@ -31,7 +32,7 @@ BUILD_ARTIFACTS = ( ) -class IdeEventHandler: +class IdeHandler: keys = ['ide'] # pylint: disable=too-many-arguments,anomalous-backslash-in-string """ diff --git a/lib/tfw/components/pipe_io/__init__.py b/lib/tfw/components/pipe_io/__init__.py new file mode 100644 index 0000000..43a5b98 --- /dev/null +++ b/lib/tfw/components/pipe_io/__init__.py @@ -0,0 +1 @@ +from .pipe_io_handler import PipeIOHandler, PipeIOHandlerBase, TransformerPipeIOHandler, CommandHandler diff --git a/lib/tfw/builtins/pipe_io_event_handler.py b/lib/tfw/components/pipe_io/pipe_io_handler.py similarity index 94% rename from lib/tfw/builtins/pipe_io_event_handler.py rename to lib/tfw/components/pipe_io/pipe_io_handler.py index 0a8e02f..4f6a850 100644 --- a/lib/tfw/builtins/pipe_io_event_handler.py +++ b/lib/tfw/components/pipe_io/pipe_io_handler.py @@ -10,14 +10,14 @@ from secrets import token_urlsafe from threading import Thread from contextlib import suppress -from tfw.components.pipe_io_server import PipeIOServer, terminate_process_on_failure +from .pipe_io_server import PipeIOServer, terminate_process_on_failure LOG = logging.getLogger(__name__) DEFAULT_PERMISSIONS = 0o600 -class PipeIOEventHandlerBase: +class PipeIOHandlerBase: keys = [''] def __init__(self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS): @@ -50,7 +50,7 @@ class CallbackPipeIOServer(PipeIOServer): LOG.exception('Failed to handle message %s from pipe %s!', message, self.in_pipe) -class PipeIOEventHandler(PipeIOEventHandlerBase): +class PipeIOHandler(PipeIOHandlerBase): def handle_event(self, message, _): json_bytes = dumps(message).encode() self.pipe_io.send_message(json_bytes) @@ -60,7 +60,7 @@ class PipeIOEventHandler(PipeIOEventHandlerBase): self.server_connector.send_message(json) -class TransformerPipeIOEventHandler(PipeIOEventHandlerBase): +class TransformerPipeIOHandler(PipeIOHandlerBase): # pylint: disable=too-many-arguments def __init__( self, in_pipe_path, out_pipe_path, @@ -97,7 +97,7 @@ class TransformerPipeIOEventHandler(PipeIOEventHandlerBase): self.server_connector.send_message(json_message) -class CommandEventHandler(PipeIOEventHandler): +class CommandHandler(PipeIOHandler): def __init__(self, command, permissions=DEFAULT_PERMISSIONS): super().__init__( self._generate_tempfilename(), diff --git a/lib/tfw/components/pipe_io_server/__init__.py b/lib/tfw/components/pipe_io/pipe_io_server/__init__.py similarity index 100% rename from lib/tfw/components/pipe_io_server/__init__.py rename to lib/tfw/components/pipe_io/pipe_io_server/__init__.py diff --git a/lib/tfw/components/pipe_io_server/deque.py b/lib/tfw/components/pipe_io/pipe_io_server/deque.py similarity index 100% rename from lib/tfw/components/pipe_io_server/deque.py rename to lib/tfw/components/pipe_io/pipe_io_server/deque.py diff --git a/lib/tfw/components/pipe_io_server/pipe.py b/lib/tfw/components/pipe_io/pipe_io_server/pipe.py similarity index 100% rename from lib/tfw/components/pipe_io_server/pipe.py rename to lib/tfw/components/pipe_io/pipe_io_server/pipe.py diff --git a/lib/tfw/components/pipe_io_server/pipe_io_server.py b/lib/tfw/components/pipe_io/pipe_io_server/pipe_io_server.py similarity index 100% rename from lib/tfw/components/pipe_io_server/pipe_io_server.py rename to lib/tfw/components/pipe_io/pipe_io_server/pipe_io_server.py diff --git a/lib/tfw/components/pipe_io_server/pipe_reader_thread.py b/lib/tfw/components/pipe_io/pipe_io_server/pipe_reader_thread.py similarity index 100% rename from lib/tfw/components/pipe_io_server/pipe_reader_thread.py rename to lib/tfw/components/pipe_io/pipe_io_server/pipe_reader_thread.py diff --git a/lib/tfw/components/pipe_io_server/pipe_writer_thread.py b/lib/tfw/components/pipe_io/pipe_io_server/pipe_writer_thread.py similarity index 100% rename from lib/tfw/components/pipe_io_server/pipe_writer_thread.py rename to lib/tfw/components/pipe_io/pipe_io_server/pipe_writer_thread.py diff --git a/lib/tfw/components/pipe_io_server/terminate_process_on_failure.py b/lib/tfw/components/pipe_io/pipe_io_server/terminate_process_on_failure.py similarity index 100% rename from lib/tfw/components/pipe_io_server/terminate_process_on_failure.py rename to lib/tfw/components/pipe_io/pipe_io_server/terminate_process_on_failure.py diff --git a/lib/tfw/components/process_management/__init__.py b/lib/tfw/components/process_management/__init__.py new file mode 100644 index 0000000..8549b02 --- /dev/null +++ b/lib/tfw/components/process_management/__init__.py @@ -0,0 +1,2 @@ +from .process_handler import ProcessHandler +from .process_log_handler import ProcessLogHandler diff --git a/lib/tfw/components/log_inotify_observer.py b/lib/tfw/components/process_management/log_inotify_observer.py similarity index 84% rename from lib/tfw/components/log_inotify_observer.py rename to lib/tfw/components/process_management/log_inotify_observer.py index 4a21f08..94400ca 100644 --- a/lib/tfw/components/log_inotify_observer.py +++ b/lib/tfw/components/process_management/log_inotify_observer.py @@ -1,19 +1,19 @@ import logging -from tfw.networking import Scope +from tfw.internals.networking import Scope +from tfw.internals.inotify import InotifyObserver -from .inotify import InotifyObserver -from .supervisor import LogManager +from .supervisor import ProcessLogManager -class LogInotifyObserver(InotifyObserver, LogManager): +class LogInotifyObserver(InotifyObserver, ProcessLogManager): def __init__(self, server_connector, supervisor_uri, process_name, log_tail=0): self._prevent_log_recursion() self._server_connector = server_connector self._process_name = process_name self.log_tail = log_tail self._procinfo = None - LogManager.__init__(self, supervisor_uri) + ProcessLogManager.__init__(self, supervisor_uri) InotifyObserver.__init__(self, self._get_logfiles()) @staticmethod diff --git a/lib/tfw/builtins/process_managing_event_handler.py b/lib/tfw/components/process_management/process_handler.py similarity index 88% rename from lib/tfw/builtins/process_managing_event_handler.py rename to lib/tfw/components/process_management/process_handler.py index 41eb114..54928fd 100644 --- a/lib/tfw/builtins/process_managing_event_handler.py +++ b/lib/tfw/components/process_management/process_handler.py @@ -1,14 +1,15 @@ import logging from xmlrpc.client import Fault as SupervisorFault -from tfw.networking import Scope -from tfw.components import ProcessManager, LogManager +from tfw.internals.networking import Scope + +from .supervisor import ProcessManager, ProcessLogManager LOG = logging.getLogger(__name__) -class ProcessManagingEventHandler(ProcessManager, LogManager): +class ProcessHandler(ProcessManager, ProcessLogManager): keys = ['processmanager'] """ Event handler that can manage processes managed by supervisor. @@ -24,7 +25,7 @@ class ProcessManagingEventHandler(ProcessManager, LogManager): """ def __init__(self, *, supervisor_uri, log_tail=0): ProcessManager.__init__(self, supervisor_uri) - LogManager.__init__(self, supervisor_uri) + ProcessLogManager.__init__(self, supervisor_uri) self.log_tail = log_tail self.commands = { 'start': self.start_process, diff --git a/lib/tfw/builtins/log_monitoring_event_handler.py b/lib/tfw/components/process_management/process_log_handler.py similarity index 96% rename from lib/tfw/builtins/log_monitoring_event_handler.py rename to lib/tfw/components/process_management/process_log_handler.py index 54927eb..4ee084a 100644 --- a/lib/tfw/builtins/log_monitoring_event_handler.py +++ b/lib/tfw/components/process_management/process_log_handler.py @@ -1,12 +1,12 @@ import logging -from tfw.components import LogInotifyObserver +from .log_inotify_observer import LogInotifyObserver LOG = logging.getLogger(__name__) -class LogMonitoringEventHandler: +class ProcessLogHandler: keys = ['logmonitor'] """ Monitors the output of a supervisor process (stdout, stderr) and diff --git a/lib/tfw/components/supervisor.py b/lib/tfw/components/process_management/supervisor.py similarity index 96% rename from lib/tfw/components/supervisor.py rename to lib/tfw/components/process_management/supervisor.py index 83e8bc4..1e50d05 100644 --- a/lib/tfw/components/supervisor.py +++ b/lib/tfw/components/process_management/supervisor.py @@ -22,7 +22,7 @@ class ProcessManager(SupervisorBase): self.start_process(process_name) -class LogManager(SupervisorBase): +class ProcessLogManager(SupervisorBase): def read_stdout(self, process_name, tail=0): return self.supervisor.readProcessStdoutLog(process_name, -tail, 0) diff --git a/lib/tfw/components/snapshots/__init__.py b/lib/tfw/components/snapshots/__init__.py new file mode 100644 index 0000000..b9b3243 --- /dev/null +++ b/lib/tfw/components/snapshots/__init__.py @@ -0,0 +1 @@ +from .snapshot_handler import SnapshotHandler diff --git a/lib/tfw/builtins/directory_snapshotting_event_handler.py b/lib/tfw/components/snapshots/snapshot_handler.py similarity index 95% rename from lib/tfw/builtins/directory_snapshotting_event_handler.py rename to lib/tfw/components/snapshots/snapshot_handler.py index 91aff92..2ca9d28 100644 --- a/lib/tfw/builtins/directory_snapshotting_event_handler.py +++ b/lib/tfw/components/snapshots/snapshot_handler.py @@ -6,14 +6,15 @@ from datetime import datetime from dateutil import parser as dateparser -from tfw.components.snapshot_provider import SnapshotProvider -from tfw.networking import Scope +from tfw.internals.networking import Scope + +from .snapshot_provider import SnapshotProvider LOG = logging.getLogger(__name__) -class DirectorySnapshottingEventHandler: +class SnapshotHandler: keys = ['snapshot'] def __init__(self, *, directories, snapshots_dir, exclude_unix_patterns=None): diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshots/snapshot_provider.py similarity index 100% rename from lib/tfw/components/snapshot_provider.py rename to lib/tfw/components/snapshots/snapshot_provider.py diff --git a/lib/tfw/components/terminal/__init__.py b/lib/tfw/components/terminal/__init__.py new file mode 100644 index 0000000..584df0d --- /dev/null +++ b/lib/tfw/components/terminal/__init__.py @@ -0,0 +1,3 @@ +from .terminal_handler import TerminalHandler +from .terminal_commands_handler import TerminalCommandsHandler +from .commands_equal import CommandsEqual diff --git a/lib/tfw/components/commands_equal.py b/lib/tfw/components/terminal/commands_equal.py similarity index 98% rename from lib/tfw/components/commands_equal.py rename to lib/tfw/components/terminal/commands_equal.py index b52aa6b..2e00d16 100644 --- a/lib/tfw/components/commands_equal.py +++ b/lib/tfw/components/terminal/commands_equal.py @@ -1,7 +1,7 @@ from shlex import split from re import search -from tfw.decorators.lazy_property import lazy_property +from tfw.internals.lazy import lazy_property class CommandsEqual: diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/terminal/history_monitor.py similarity index 98% rename from lib/tfw/components/history_monitor.py rename to lib/tfw/components/terminal/history_monitor.py index 899217d..26b230d 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/terminal/history_monitor.py @@ -2,7 +2,7 @@ from re import findall from re import compile as compileregex from abc import ABC, abstractmethod -from tfw.components.inotify import InotifyObserver +from tfw.internals.inotify import InotifyObserver class HistoryMonitor(ABC, InotifyObserver): diff --git a/lib/tfw/components/terminado_mini_server.py b/lib/tfw/components/terminal/terminado_mini_server.py similarity index 100% rename from lib/tfw/components/terminado_mini_server.py rename to lib/tfw/components/terminal/terminado_mini_server.py diff --git a/lib/tfw/components/terminal_commands.py b/lib/tfw/components/terminal/terminal_commands.py similarity index 100% rename from lib/tfw/components/terminal_commands.py rename to lib/tfw/components/terminal/terminal_commands.py diff --git a/lib/tfw/builtins/terminal_commands_event_handler.py b/lib/tfw/components/terminal/terminal_commands_handler.py similarity index 58% rename from lib/tfw/builtins/terminal_commands_event_handler.py rename to lib/tfw/components/terminal/terminal_commands_handler.py index 2bb7399..dcfd5f0 100644 --- a/lib/tfw/builtins/terminal_commands_event_handler.py +++ b/lib/tfw/components/terminal/terminal_commands_handler.py @@ -1,7 +1,7 @@ -from tfw.components import TerminalCommands +from .terminal_commands import TerminalCommands -class TerminalCommandsEventHandler(TerminalCommands): +class TerminalCommandsHandler(TerminalCommands): keys = ['history.bash'] def handle_event(self, message, _): diff --git a/lib/tfw/builtins/terminal_event_handler.py b/lib/tfw/components/terminal/terminal_handler.py similarity index 95% rename from lib/tfw/builtins/terminal_event_handler.py rename to lib/tfw/components/terminal/terminal_handler.py index 6cee755..2cbdff4 100644 --- a/lib/tfw/builtins/terminal_event_handler.py +++ b/lib/tfw/components/terminal/terminal_handler.py @@ -1,12 +1,13 @@ import logging -from tfw.components import BashMonitor, TerminadoMiniServer +from .history_monitor import BashMonitor +from .terminado_mini_server import TerminadoMiniServer LOG = logging.getLogger(__name__) -class TerminalEventHandler: +class TerminalHandler: keys = ['shell'] """ Event handler responsible for managing terminal sessions for frontend xterm diff --git a/lib/tfw/event_handlers.py b/lib/tfw/event_handlers.py new file mode 100644 index 0000000..8dc872e --- /dev/null +++ b/lib/tfw/event_handlers.py @@ -0,0 +1,2 @@ +# pylint: disable=unused-import +from tfw.internals.event_handling import EventHandler, FSMAwareEventHandler diff --git a/lib/tfw/fsm/fsm_base.py b/lib/tfw/fsm/fsm_base.py index e4f9645..0fac40b 100644 --- a/lib/tfw/fsm/fsm_base.py +++ b/lib/tfw/fsm/fsm_base.py @@ -4,7 +4,7 @@ from datetime import datetime from transitions import Machine, MachineError -from tfw.mixins.callback_mixin import CallbackMixin +from tfw.internals.callback_mixin import CallbackMixin LOG = logging.getLogger(__name__) diff --git a/lib/tfw/fsm/linear_fsm.py b/lib/tfw/fsm/linear_fsm.py index 383f98e..354ffd2 100644 --- a/lib/tfw/fsm/linear_fsm.py +++ b/lib/tfw/fsm/linear_fsm.py @@ -1,6 +1,6 @@ from transitions import State -from tfw.fsm.fsm_base import FSMBase +from .fsm_base import FSMBase class LinearFSM(FSMBase): diff --git a/lib/tfw/fsm/yaml_fsm.py b/lib/tfw/fsm/yaml_fsm.py index facce7c..32c002b 100644 --- a/lib/tfw/fsm/yaml_fsm.py +++ b/lib/tfw/fsm/yaml_fsm.py @@ -6,7 +6,7 @@ import yaml import jinja2 from transitions import State -from tfw.fsm.fsm_base import FSMBase +from .fsm_base import FSMBase class YamlFSM(FSMBase): diff --git a/lib/tfw/decorators/__init__.py b/lib/tfw/internals/__init__.py similarity index 100% rename from lib/tfw/decorators/__init__.py rename to lib/tfw/internals/__init__.py diff --git a/lib/tfw/mixins/callback_mixin.py b/lib/tfw/internals/callback_mixin.py similarity index 95% rename from lib/tfw/mixins/callback_mixin.py rename to lib/tfw/internals/callback_mixin.py index 36b0f64..c54db5b 100644 --- a/lib/tfw/mixins/callback_mixin.py +++ b/lib/tfw/internals/callback_mixin.py @@ -1,6 +1,6 @@ from functools import partial -from tfw.decorators.lazy_property import lazy_property +from .lazy import lazy_property class CallbackMixin: diff --git a/lib/tfw/crypto.py b/lib/tfw/internals/crypto.py similarity index 96% rename from lib/tfw/crypto.py rename to lib/tfw/internals/crypto.py index 562420b..04aff16 100644 --- a/lib/tfw/crypto.py +++ b/lib/tfw/internals/crypto.py @@ -11,8 +11,8 @@ from cryptography.hazmat.primitives.hashes import SHA256 from cryptography.hazmat.primitives.hmac import HMAC as _HMAC from cryptography.exceptions import InvalidSignature -from tfw.networking import message_bytes -from tfw.decorators.lazy_property import lazy_property +from tfw.internals.networking import message_bytes +from tfw.internals.lazy import lazy_property from tfw.config import TFWENV diff --git a/lib/tfw/event_handlers/__init__.py b/lib/tfw/internals/event_handling/__init__.py similarity index 100% rename from lib/tfw/event_handlers/__init__.py rename to lib/tfw/internals/event_handling/__init__.py diff --git a/lib/tfw/event_handlers/event_handler.py b/lib/tfw/internals/event_handling/event_handler.py similarity index 100% rename from lib/tfw/event_handlers/event_handler.py rename to lib/tfw/internals/event_handling/event_handler.py diff --git a/lib/tfw/event_handlers/event_handler_factory_base.py b/lib/tfw/internals/event_handling/event_handler_factory_base.py similarity index 100% rename from lib/tfw/event_handlers/event_handler_factory_base.py rename to lib/tfw/internals/event_handling/event_handler_factory_base.py diff --git a/lib/tfw/event_handlers/fsm_aware.py b/lib/tfw/internals/event_handling/fsm_aware.py similarity index 94% rename from lib/tfw/event_handlers/fsm_aware.py rename to lib/tfw/internals/event_handling/fsm_aware.py index 76d8d3f..5254e68 100644 --- a/lib/tfw/event_handlers/fsm_aware.py +++ b/lib/tfw/internals/event_handling/fsm_aware.py @@ -1,6 +1,6 @@ import logging -from tfw.crypto import KeyManager, verify_message +from tfw.internals.crypto import KeyManager, verify_message LOG = logging.getLogger(__name__) diff --git a/lib/tfw/event_handlers/fsm_aware_event_handler.py b/lib/tfw/internals/event_handling/fsm_aware_event_handler.py similarity index 100% rename from lib/tfw/event_handlers/fsm_aware_event_handler.py rename to lib/tfw/internals/event_handling/fsm_aware_event_handler.py diff --git a/lib/tfw/event_handlers/test_event_handler.py b/lib/tfw/internals/event_handling/test_event_handler.py similarity index 100% rename from lib/tfw/event_handlers/test_event_handler.py rename to lib/tfw/internals/event_handling/test_event_handler.py diff --git a/lib/tfw/components/inotify/__init__.py b/lib/tfw/internals/inotify/__init__.py similarity index 100% rename from lib/tfw/components/inotify/__init__.py rename to lib/tfw/internals/inotify/__init__.py diff --git a/lib/tfw/components/inotify/inotify.py b/lib/tfw/internals/inotify/inotify.py similarity index 100% rename from lib/tfw/components/inotify/inotify.py rename to lib/tfw/internals/inotify/inotify.py diff --git a/lib/tfw/components/inotify/test_inotify.py b/lib/tfw/internals/inotify/test_inotify.py similarity index 100% rename from lib/tfw/components/inotify/test_inotify.py rename to lib/tfw/internals/inotify/test_inotify.py diff --git a/lib/tfw/decorators/lazy_property.py b/lib/tfw/internals/lazy.py similarity index 100% rename from lib/tfw/decorators/lazy_property.py rename to lib/tfw/internals/lazy.py diff --git a/lib/tfw/networking/__init__.py b/lib/tfw/internals/networking/__init__.py similarity index 100% rename from lib/tfw/networking/__init__.py rename to lib/tfw/internals/networking/__init__.py diff --git a/lib/tfw/networking/event_handler_connector.py b/lib/tfw/internals/networking/event_handler_connector.py similarity index 100% rename from lib/tfw/networking/event_handler_connector.py rename to lib/tfw/internals/networking/event_handler_connector.py diff --git a/lib/tfw/networking/scope.py b/lib/tfw/internals/networking/scope.py similarity index 100% rename from lib/tfw/networking/scope.py rename to lib/tfw/internals/networking/scope.py diff --git a/lib/tfw/networking/serialization.py b/lib/tfw/internals/networking/serialization.py similarity index 100% rename from lib/tfw/networking/serialization.py rename to lib/tfw/internals/networking/serialization.py diff --git a/lib/tfw/networking/server_connector.py b/lib/tfw/internals/networking/server_connector.py similarity index 100% rename from lib/tfw/networking/server_connector.py rename to lib/tfw/internals/networking/server_connector.py diff --git a/lib/tfw/internals/server/__init__.py b/lib/tfw/internals/server/__init__.py new file mode 100644 index 0000000..73e3f40 --- /dev/null +++ b/lib/tfw/internals/server/__init__.py @@ -0,0 +1 @@ +from .zmq_websocket_router import ZMQWebSocketRouter diff --git a/lib/tfw/server/zmq_websocket_router.py b/lib/tfw/internals/server/zmq_websocket_router.py similarity index 97% rename from lib/tfw/server/zmq_websocket_router.py rename to lib/tfw/internals/server/zmq_websocket_router.py index b7adf16..cdafc2f 100644 --- a/lib/tfw/server/zmq_websocket_router.py +++ b/lib/tfw/internals/server/zmq_websocket_router.py @@ -3,7 +3,7 @@ import logging from tornado.websocket import WebSocketHandler -from tfw.networking import Scope +from tfw.internals.networking import Scope LOG = logging.getLogger(__name__) diff --git a/lib/tfw/logging.py b/lib/tfw/logging.py index d405adb..dad7499 100644 --- a/lib/tfw/logging.py +++ b/lib/tfw/logging.py @@ -102,7 +102,7 @@ class LogFormatter(Formatter): class VerboseLogFormatter(Formatter): - def format(self, record): + def format(self, record): # pylint: disable=no-self-use date = datetime.utcfromtimestamp(record.created).strftime('%H:%M:%S') if record.args: message = record.msg % record.args diff --git a/lib/tfw/main/__init__.py b/lib/tfw/main/__init__.py index 9e1c892..3e731f9 100644 --- a/lib/tfw/main/__init__.py +++ b/lib/tfw/main/__init__.py @@ -1,3 +1,4 @@ from .tfw_connector import TFWUplinkConnector, TFWConnector from .event_handler_factory import EventHandlerFactory from .signal_handling import setup_signal_handlers +from .tfw_server import TFWServer diff --git a/lib/tfw/main/event_handler_factory.py b/lib/tfw/main/event_handler_factory.py index 271be8c..c2c4045 100644 --- a/lib/tfw/main/event_handler_factory.py +++ b/lib/tfw/main/event_handler_factory.py @@ -1,4 +1,4 @@ -from tfw.event_handlers import EventHandlerFactoryBase +from tfw.internals.event_handling import EventHandlerFactoryBase from .tfw_connector import TFWConnector diff --git a/lib/tfw/main/signal_handling.py b/lib/tfw/main/signal_handling.py index a8750ef..387c77d 100644 --- a/lib/tfw/main/signal_handling.py +++ b/lib/tfw/main/signal_handling.py @@ -1,6 +1,6 @@ from signal import signal, SIGTERM, SIGINT -from tfw.event_handlers import EventHandler +from tfw.internals.event_handling import EventHandler def setup_signal_handlers(): diff --git a/lib/tfw/main/tfw_connector.py b/lib/tfw/main/tfw_connector.py index 2e56996..5324ee6 100644 --- a/lib/tfw/main/tfw_connector.py +++ b/lib/tfw/main/tfw_connector.py @@ -1,4 +1,4 @@ -from tfw.networking import ServerConnector, ServerUplinkConnector +from tfw.internals.networking import ServerConnector, ServerUplinkConnector from tfw.config import TFWENV diff --git a/lib/tfw/server/tfw_server.py b/lib/tfw/main/tfw_server.py similarity index 88% rename from lib/tfw/server/tfw_server.py rename to lib/tfw/main/tfw_server.py index d524265..22d216f 100644 --- a/lib/tfw/server/tfw_server.py +++ b/lib/tfw/main/tfw_server.py @@ -2,10 +2,10 @@ import logging from tornado.web import Application -from tfw.networking import EventHandlerConnector +from tfw.internals.networking import EventHandlerConnector +from tfw.internals.server import ZMQWebSocketRouter from tfw.config import TFWENV -from .zmq_websocket_router import ZMQWebSocketRouter LOG = logging.getLogger(__name__) diff --git a/lib/tfw/mixins/__init__.py b/lib/tfw/mixins/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/tfw/server/__init__.py b/lib/tfw/server/__init__.py deleted file mode 100644 index e2c01a9..0000000 --- a/lib/tfw/server/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .tfw_server import TFWServer diff --git a/supervisor/tfw_server.py b/supervisor/tfw_server.py index 812873c..f3ad4be 100644 --- a/supervisor/tfw_server.py +++ b/supervisor/tfw_server.py @@ -2,7 +2,7 @@ from sys import stderr from tornado.ioloop import IOLoop -from tfw.server import TFWServer +from tfw.main import TFWServer from tfw.config import TFWENV from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter From 52399f413ca4e8754dc14645d8f1f3af14e24d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 24 Jul 2019 15:50:41 +0200 Subject: [PATCH 089/174] Simplify package structure --- Dockerfile | 6 ++--- lib/tao/config/__init__.py | 1 - lib/tao/config/envvars.py | 3 --- lib/tfw/config/__init__.py | 1 - lib/tfw/config/envvars.py | 3 --- lib/tfw/internals/__init__.py | 0 setup.py | 26 +++++++++---------- {lib/tao => tfw}/__init__.py | 0 {lib/tfw => tfw/components}/__init__.py | 0 .../components/frontend/__init__.py | 0 .../components/frontend/frontend_handler.py | 0 .../components/frontend/message_sender.py | 0 .../components/frontend/message_storage.py | 0 {lib/tfw => tfw}/components/fsm/__init__.py | 0 .../tfw => tfw}/components/fsm/fsm_handler.py | 0 .../tfw => tfw}/components/fsm/fsm_updater.py | 0 {lib/tfw => tfw}/components/ide/__init__.py | 0 .../components/ide/file_manager/__init__.py | 0 .../ide/file_manager/file_manager.py | 0 .../ide/file_manager/test_file_manager.py | 0 .../tfw => tfw}/components/ide/ide_handler.py | 0 .../components/pipe_io/__init__.py | 0 .../components/pipe_io/pipe_io_handler.py | 0 .../pipe_io/pipe_io_server/__init__.py | 0 .../pipe_io/pipe_io_server/deque.py | 0 .../components/pipe_io/pipe_io_server/pipe.py | 0 .../pipe_io/pipe_io_server/pipe_io_server.py | 0 .../pipe_io_server/pipe_reader_thread.py | 0 .../pipe_io_server/pipe_writer_thread.py | 0 .../terminate_process_on_failure.py | 0 .../components/process_management/__init__.py | 0 .../log_inotify_observer.py | 0 .../process_management/process_handler.py | 0 .../process_management/process_log_handler.py | 0 .../process_management/supervisor.py | 0 .../components/snapshots/__init__.py | 0 .../components/snapshots/snapshot_handler.py | 0 .../components/snapshots/snapshot_provider.py | 0 .../components/terminal/__init__.py | 0 .../components/terminal/commands_equal.py | 0 .../components/terminal/history_monitor.py | 0 .../terminal/terminado_mini_server.py | 0 .../components/terminal/terminal_commands.py | 0 .../terminal/terminal_commands_handler.py | 0 .../components/terminal/terminal_handler.py | 0 tfw/config/__init__.py | 1 + tfw/config/envvars.py | 5 ++++ .../config/lazy_environment.py | 0 {lib/tfw => tfw}/event_handlers.py | 0 {lib/tfw => tfw}/fsm/__init__.py | 0 {lib/tfw => tfw}/fsm/fsm_base.py | 0 {lib/tfw => tfw}/fsm/linear_fsm.py | 0 {lib/tfw => tfw}/fsm/yaml_fsm.py | 0 .../components => tfw/internals}/__init__.py | 0 {lib/tfw => tfw}/internals/callback_mixin.py | 0 {lib/tfw => tfw}/internals/crypto.py | 0 .../internals/event_handling/__init__.py | 0 .../internals/event_handling/event_handler.py | 0 .../event_handler_factory_base.py | 0 .../internals/event_handling/fsm_aware.py | 0 .../event_handling/fsm_aware_event_handler.py | 0 .../event_handling/test_event_handler.py | 0 .../tfw => tfw}/internals/inotify/__init__.py | 0 {lib/tfw => tfw}/internals/inotify/inotify.py | 0 .../internals/inotify/test_inotify.py | 0 {lib/tfw => tfw}/internals/lazy.py | 0 .../internals/networking/__init__.py | 0 .../networking/event_handler_connector.py | 0 .../tfw => tfw}/internals/networking/scope.py | 0 .../internals/networking/serialization.py | 0 .../internals/networking/server_connector.py | 0 {lib/tfw => tfw}/internals/server/__init__.py | 0 .../internals/server/zmq_websocket_router.py | 0 {lib/tfw => tfw}/logging.py | 0 {lib/tfw => tfw}/main/__init__.py | 0 .../tfw => tfw}/main/event_handler_factory.py | 0 {lib/tfw => tfw}/main/signal_handling.py | 0 {lib/tfw => tfw}/main/tfw_connector.py | 0 {lib/tfw => tfw}/main/tfw_server.py | 0 79 files changed, 22 insertions(+), 24 deletions(-) delete mode 100644 lib/tao/config/__init__.py delete mode 100644 lib/tao/config/envvars.py delete mode 100644 lib/tfw/config/__init__.py delete mode 100644 lib/tfw/config/envvars.py delete mode 100644 lib/tfw/internals/__init__.py rename {lib/tao => tfw}/__init__.py (100%) rename {lib/tfw => tfw/components}/__init__.py (100%) rename {lib/tfw => tfw}/components/frontend/__init__.py (100%) rename {lib/tfw => tfw}/components/frontend/frontend_handler.py (100%) rename {lib/tfw => tfw}/components/frontend/message_sender.py (100%) rename {lib/tfw => tfw}/components/frontend/message_storage.py (100%) rename {lib/tfw => tfw}/components/fsm/__init__.py (100%) rename {lib/tfw => tfw}/components/fsm/fsm_handler.py (100%) rename {lib/tfw => tfw}/components/fsm/fsm_updater.py (100%) rename {lib/tfw => tfw}/components/ide/__init__.py (100%) rename {lib/tfw => tfw}/components/ide/file_manager/__init__.py (100%) rename {lib/tfw => tfw}/components/ide/file_manager/file_manager.py (100%) rename {lib/tfw => tfw}/components/ide/file_manager/test_file_manager.py (100%) rename {lib/tfw => tfw}/components/ide/ide_handler.py (100%) rename {lib/tfw => tfw}/components/pipe_io/__init__.py (100%) rename {lib/tfw => tfw}/components/pipe_io/pipe_io_handler.py (100%) rename {lib/tfw => tfw}/components/pipe_io/pipe_io_server/__init__.py (100%) rename {lib/tfw => tfw}/components/pipe_io/pipe_io_server/deque.py (100%) rename {lib/tfw => tfw}/components/pipe_io/pipe_io_server/pipe.py (100%) rename {lib/tfw => tfw}/components/pipe_io/pipe_io_server/pipe_io_server.py (100%) rename {lib/tfw => tfw}/components/pipe_io/pipe_io_server/pipe_reader_thread.py (100%) rename {lib/tfw => tfw}/components/pipe_io/pipe_io_server/pipe_writer_thread.py (100%) rename {lib/tfw => tfw}/components/pipe_io/pipe_io_server/terminate_process_on_failure.py (100%) rename {lib/tfw => tfw}/components/process_management/__init__.py (100%) rename {lib/tfw => tfw}/components/process_management/log_inotify_observer.py (100%) rename {lib/tfw => tfw}/components/process_management/process_handler.py (100%) rename {lib/tfw => tfw}/components/process_management/process_log_handler.py (100%) rename {lib/tfw => tfw}/components/process_management/supervisor.py (100%) rename {lib/tfw => tfw}/components/snapshots/__init__.py (100%) rename {lib/tfw => tfw}/components/snapshots/snapshot_handler.py (100%) rename {lib/tfw => tfw}/components/snapshots/snapshot_provider.py (100%) rename {lib/tfw => tfw}/components/terminal/__init__.py (100%) rename {lib/tfw => tfw}/components/terminal/commands_equal.py (100%) rename {lib/tfw => tfw}/components/terminal/history_monitor.py (100%) rename {lib/tfw => tfw}/components/terminal/terminado_mini_server.py (100%) rename {lib/tfw => tfw}/components/terminal/terminal_commands.py (100%) rename {lib/tfw => tfw}/components/terminal/terminal_commands_handler.py (100%) rename {lib/tfw => tfw}/components/terminal/terminal_handler.py (100%) create mode 100644 tfw/config/__init__.py create mode 100644 tfw/config/envvars.py rename lib/envvars/__init__.py => tfw/config/lazy_environment.py (100%) rename {lib/tfw => tfw}/event_handlers.py (100%) rename {lib/tfw => tfw}/fsm/__init__.py (100%) rename {lib/tfw => tfw}/fsm/fsm_base.py (100%) rename {lib/tfw => tfw}/fsm/linear_fsm.py (100%) rename {lib/tfw => tfw}/fsm/yaml_fsm.py (100%) rename {lib/tfw/components => tfw/internals}/__init__.py (100%) rename {lib/tfw => tfw}/internals/callback_mixin.py (100%) rename {lib/tfw => tfw}/internals/crypto.py (100%) rename {lib/tfw => tfw}/internals/event_handling/__init__.py (100%) rename {lib/tfw => tfw}/internals/event_handling/event_handler.py (100%) rename {lib/tfw => tfw}/internals/event_handling/event_handler_factory_base.py (100%) rename {lib/tfw => tfw}/internals/event_handling/fsm_aware.py (100%) rename {lib/tfw => tfw}/internals/event_handling/fsm_aware_event_handler.py (100%) rename {lib/tfw => tfw}/internals/event_handling/test_event_handler.py (100%) rename {lib/tfw => tfw}/internals/inotify/__init__.py (100%) rename {lib/tfw => tfw}/internals/inotify/inotify.py (100%) rename {lib/tfw => tfw}/internals/inotify/test_inotify.py (100%) rename {lib/tfw => tfw}/internals/lazy.py (100%) rename {lib/tfw => tfw}/internals/networking/__init__.py (100%) rename {lib/tfw => tfw}/internals/networking/event_handler_connector.py (100%) rename {lib/tfw => tfw}/internals/networking/scope.py (100%) rename {lib/tfw => tfw}/internals/networking/serialization.py (100%) rename {lib/tfw => tfw}/internals/networking/server_connector.py (100%) rename {lib/tfw => tfw}/internals/server/__init__.py (100%) rename {lib/tfw => tfw}/internals/server/zmq_websocket_router.py (100%) rename {lib/tfw => tfw}/logging.py (100%) rename {lib/tfw => tfw}/main/__init__.py (100%) rename {lib/tfw => tfw}/main/event_handler_factory.py (100%) rename {lib/tfw => tfw}/main/signal_handling.py (100%) rename {lib/tfw => tfw}/main/tfw_connector.py (100%) rename {lib/tfw => tfw}/main/tfw_server.py (100%) diff --git a/Dockerfile b/Dockerfile index 1290e10..5df5cf4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,10 +54,10 @@ COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS} COPY nginx/nginx.conf ${TFW_NGINX_CONF} COPY nginx/default.conf ${TFW_NGINX_DEFAULT} COPY nginx/components/ ${TFW_NGINX_COMPONENTS} -COPY lib ${TFW_LIB_DIR}/ +COPY tfw ${TFW_LIB_DIR}/tfw COPY supervisor/tfw_server.py ${TFW_SERVER_DIR}/ -RUN for dir in "${TFW_LIB_DIR}"/{tfw,tao,envvars} "/etc/nginx" "/etc/supervisor"; do \ +RUN for dir in "${TFW_LIB_DIR}"/tfw "/etc/nginx" "/etc/supervisor"; do \ chown -R root:root "$dir" && chmod -R 700 "$dir"; \ done @@ -70,7 +70,7 @@ ONBUILD COPY ${BUILD_CONTEXT}/supervisor/ ${TFW_SUPERVISORD_COMPONENTS} ONBUILD RUN for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do \ envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f~ && mv $f~ $f ;\ done -ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx", "${TFW_LIB_DIR}/envvars", "${TFW_LIB_DIR}/tfw"] +ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx", "${TFW_LIB_DIR}/tfw"] ONBUILD COPY ${BUILD_CONTEXT}/frontend /data/ ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn install --frozen-lockfile || : diff --git a/lib/tao/config/__init__.py b/lib/tao/config/__init__.py deleted file mode 100644 index fd07e8b..0000000 --- a/lib/tao/config/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .envvars import TAOENV diff --git a/lib/tao/config/envvars.py b/lib/tao/config/envvars.py deleted file mode 100644 index 813d06f..0000000 --- a/lib/tao/config/envvars.py +++ /dev/null @@ -1,3 +0,0 @@ -from envvars import LazyEnvironment - -TAOENV = LazyEnvironment('AVATAO_', 'taoenvtuple').environment diff --git a/lib/tfw/config/__init__.py b/lib/tfw/config/__init__.py deleted file mode 100644 index b9719ef..0000000 --- a/lib/tfw/config/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .envvars import TFWENV diff --git a/lib/tfw/config/envvars.py b/lib/tfw/config/envvars.py deleted file mode 100644 index 68adb0e..0000000 --- a/lib/tfw/config/envvars.py +++ /dev/null @@ -1,3 +0,0 @@ -from envvars import LazyEnvironment - -TFWENV = LazyEnvironment('TFW_', 'tfwenvtuple').environment diff --git a/lib/tfw/internals/__init__.py b/lib/tfw/internals/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/setup.py b/setup.py index e5fa212..c4c7912 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from os.path import dirname, realpath, join -from setuptools import setup, find_packages +from setuptools import setup here = dirname(realpath(__file__)) @@ -10,20 +10,20 @@ with open(join(here, 'requirements.txt'), 'r') as ifile: requirements = ifile.read().splitlines() setup( - name = 'tfw', - version = version, - description = 'Avatao tutorial-framework', - url = 'https://github.com/avatao-content/baseimage-tutorial-framework', - author = 'Avatao.com Innovative Learning Kft.', - author_email = 'support@avatao.com', - license = 'custom', - packages = find_packages('lib'), - package_dir = {'': 'lib'}, - install_requires = requirements, - extras_require = { + name='tfw', + version=version, + description='Avatao tutorial-framework', + url='https://github.com/avatao-content/baseimage-tutorial-framework', + author='Avatao.com Innovative Learning Kft.', + author_email='support@avatao.com', + license='custom', + packages=['tfw'], + package_dir={'tfw': 'tfw'}, + install_requires=requirements, + extras_require={ 'docs': [ 'sphinx >= 1.7.0', ], }, - zip_safe = False, + zip_safe=False, ) diff --git a/lib/tao/__init__.py b/tfw/__init__.py similarity index 100% rename from lib/tao/__init__.py rename to tfw/__init__.py diff --git a/lib/tfw/__init__.py b/tfw/components/__init__.py similarity index 100% rename from lib/tfw/__init__.py rename to tfw/components/__init__.py diff --git a/lib/tfw/components/frontend/__init__.py b/tfw/components/frontend/__init__.py similarity index 100% rename from lib/tfw/components/frontend/__init__.py rename to tfw/components/frontend/__init__.py diff --git a/lib/tfw/components/frontend/frontend_handler.py b/tfw/components/frontend/frontend_handler.py similarity index 100% rename from lib/tfw/components/frontend/frontend_handler.py rename to tfw/components/frontend/frontend_handler.py diff --git a/lib/tfw/components/frontend/message_sender.py b/tfw/components/frontend/message_sender.py similarity index 100% rename from lib/tfw/components/frontend/message_sender.py rename to tfw/components/frontend/message_sender.py diff --git a/lib/tfw/components/frontend/message_storage.py b/tfw/components/frontend/message_storage.py similarity index 100% rename from lib/tfw/components/frontend/message_storage.py rename to tfw/components/frontend/message_storage.py diff --git a/lib/tfw/components/fsm/__init__.py b/tfw/components/fsm/__init__.py similarity index 100% rename from lib/tfw/components/fsm/__init__.py rename to tfw/components/fsm/__init__.py diff --git a/lib/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py similarity index 100% rename from lib/tfw/components/fsm/fsm_handler.py rename to tfw/components/fsm/fsm_handler.py diff --git a/lib/tfw/components/fsm/fsm_updater.py b/tfw/components/fsm/fsm_updater.py similarity index 100% rename from lib/tfw/components/fsm/fsm_updater.py rename to tfw/components/fsm/fsm_updater.py diff --git a/lib/tfw/components/ide/__init__.py b/tfw/components/ide/__init__.py similarity index 100% rename from lib/tfw/components/ide/__init__.py rename to tfw/components/ide/__init__.py diff --git a/lib/tfw/components/ide/file_manager/__init__.py b/tfw/components/ide/file_manager/__init__.py similarity index 100% rename from lib/tfw/components/ide/file_manager/__init__.py rename to tfw/components/ide/file_manager/__init__.py diff --git a/lib/tfw/components/ide/file_manager/file_manager.py b/tfw/components/ide/file_manager/file_manager.py similarity index 100% rename from lib/tfw/components/ide/file_manager/file_manager.py rename to tfw/components/ide/file_manager/file_manager.py diff --git a/lib/tfw/components/ide/file_manager/test_file_manager.py b/tfw/components/ide/file_manager/test_file_manager.py similarity index 100% rename from lib/tfw/components/ide/file_manager/test_file_manager.py rename to tfw/components/ide/file_manager/test_file_manager.py diff --git a/lib/tfw/components/ide/ide_handler.py b/tfw/components/ide/ide_handler.py similarity index 100% rename from lib/tfw/components/ide/ide_handler.py rename to tfw/components/ide/ide_handler.py diff --git a/lib/tfw/components/pipe_io/__init__.py b/tfw/components/pipe_io/__init__.py similarity index 100% rename from lib/tfw/components/pipe_io/__init__.py rename to tfw/components/pipe_io/__init__.py diff --git a/lib/tfw/components/pipe_io/pipe_io_handler.py b/tfw/components/pipe_io/pipe_io_handler.py similarity index 100% rename from lib/tfw/components/pipe_io/pipe_io_handler.py rename to tfw/components/pipe_io/pipe_io_handler.py diff --git a/lib/tfw/components/pipe_io/pipe_io_server/__init__.py b/tfw/components/pipe_io/pipe_io_server/__init__.py similarity index 100% rename from lib/tfw/components/pipe_io/pipe_io_server/__init__.py rename to tfw/components/pipe_io/pipe_io_server/__init__.py diff --git a/lib/tfw/components/pipe_io/pipe_io_server/deque.py b/tfw/components/pipe_io/pipe_io_server/deque.py similarity index 100% rename from lib/tfw/components/pipe_io/pipe_io_server/deque.py rename to tfw/components/pipe_io/pipe_io_server/deque.py diff --git a/lib/tfw/components/pipe_io/pipe_io_server/pipe.py b/tfw/components/pipe_io/pipe_io_server/pipe.py similarity index 100% rename from lib/tfw/components/pipe_io/pipe_io_server/pipe.py rename to tfw/components/pipe_io/pipe_io_server/pipe.py diff --git a/lib/tfw/components/pipe_io/pipe_io_server/pipe_io_server.py b/tfw/components/pipe_io/pipe_io_server/pipe_io_server.py similarity index 100% rename from lib/tfw/components/pipe_io/pipe_io_server/pipe_io_server.py rename to tfw/components/pipe_io/pipe_io_server/pipe_io_server.py diff --git a/lib/tfw/components/pipe_io/pipe_io_server/pipe_reader_thread.py b/tfw/components/pipe_io/pipe_io_server/pipe_reader_thread.py similarity index 100% rename from lib/tfw/components/pipe_io/pipe_io_server/pipe_reader_thread.py rename to tfw/components/pipe_io/pipe_io_server/pipe_reader_thread.py diff --git a/lib/tfw/components/pipe_io/pipe_io_server/pipe_writer_thread.py b/tfw/components/pipe_io/pipe_io_server/pipe_writer_thread.py similarity index 100% rename from lib/tfw/components/pipe_io/pipe_io_server/pipe_writer_thread.py rename to tfw/components/pipe_io/pipe_io_server/pipe_writer_thread.py diff --git a/lib/tfw/components/pipe_io/pipe_io_server/terminate_process_on_failure.py b/tfw/components/pipe_io/pipe_io_server/terminate_process_on_failure.py similarity index 100% rename from lib/tfw/components/pipe_io/pipe_io_server/terminate_process_on_failure.py rename to tfw/components/pipe_io/pipe_io_server/terminate_process_on_failure.py diff --git a/lib/tfw/components/process_management/__init__.py b/tfw/components/process_management/__init__.py similarity index 100% rename from lib/tfw/components/process_management/__init__.py rename to tfw/components/process_management/__init__.py diff --git a/lib/tfw/components/process_management/log_inotify_observer.py b/tfw/components/process_management/log_inotify_observer.py similarity index 100% rename from lib/tfw/components/process_management/log_inotify_observer.py rename to tfw/components/process_management/log_inotify_observer.py diff --git a/lib/tfw/components/process_management/process_handler.py b/tfw/components/process_management/process_handler.py similarity index 100% rename from lib/tfw/components/process_management/process_handler.py rename to tfw/components/process_management/process_handler.py diff --git a/lib/tfw/components/process_management/process_log_handler.py b/tfw/components/process_management/process_log_handler.py similarity index 100% rename from lib/tfw/components/process_management/process_log_handler.py rename to tfw/components/process_management/process_log_handler.py diff --git a/lib/tfw/components/process_management/supervisor.py b/tfw/components/process_management/supervisor.py similarity index 100% rename from lib/tfw/components/process_management/supervisor.py rename to tfw/components/process_management/supervisor.py diff --git a/lib/tfw/components/snapshots/__init__.py b/tfw/components/snapshots/__init__.py similarity index 100% rename from lib/tfw/components/snapshots/__init__.py rename to tfw/components/snapshots/__init__.py diff --git a/lib/tfw/components/snapshots/snapshot_handler.py b/tfw/components/snapshots/snapshot_handler.py similarity index 100% rename from lib/tfw/components/snapshots/snapshot_handler.py rename to tfw/components/snapshots/snapshot_handler.py diff --git a/lib/tfw/components/snapshots/snapshot_provider.py b/tfw/components/snapshots/snapshot_provider.py similarity index 100% rename from lib/tfw/components/snapshots/snapshot_provider.py rename to tfw/components/snapshots/snapshot_provider.py diff --git a/lib/tfw/components/terminal/__init__.py b/tfw/components/terminal/__init__.py similarity index 100% rename from lib/tfw/components/terminal/__init__.py rename to tfw/components/terminal/__init__.py diff --git a/lib/tfw/components/terminal/commands_equal.py b/tfw/components/terminal/commands_equal.py similarity index 100% rename from lib/tfw/components/terminal/commands_equal.py rename to tfw/components/terminal/commands_equal.py diff --git a/lib/tfw/components/terminal/history_monitor.py b/tfw/components/terminal/history_monitor.py similarity index 100% rename from lib/tfw/components/terminal/history_monitor.py rename to tfw/components/terminal/history_monitor.py diff --git a/lib/tfw/components/terminal/terminado_mini_server.py b/tfw/components/terminal/terminado_mini_server.py similarity index 100% rename from lib/tfw/components/terminal/terminado_mini_server.py rename to tfw/components/terminal/terminado_mini_server.py diff --git a/lib/tfw/components/terminal/terminal_commands.py b/tfw/components/terminal/terminal_commands.py similarity index 100% rename from lib/tfw/components/terminal/terminal_commands.py rename to tfw/components/terminal/terminal_commands.py diff --git a/lib/tfw/components/terminal/terminal_commands_handler.py b/tfw/components/terminal/terminal_commands_handler.py similarity index 100% rename from lib/tfw/components/terminal/terminal_commands_handler.py rename to tfw/components/terminal/terminal_commands_handler.py diff --git a/lib/tfw/components/terminal/terminal_handler.py b/tfw/components/terminal/terminal_handler.py similarity index 100% rename from lib/tfw/components/terminal/terminal_handler.py rename to tfw/components/terminal/terminal_handler.py diff --git a/tfw/config/__init__.py b/tfw/config/__init__.py new file mode 100644 index 0000000..f5476e9 --- /dev/null +++ b/tfw/config/__init__.py @@ -0,0 +1 @@ +from .envvars import TFWENV, TAOENV diff --git a/tfw/config/envvars.py b/tfw/config/envvars.py new file mode 100644 index 0000000..1e4c89a --- /dev/null +++ b/tfw/config/envvars.py @@ -0,0 +1,5 @@ +from .lazy_environment import LazyEnvironment + + +TFWENV = LazyEnvironment('TFW_', 'tfwenvtuple').environment +TAOENV = LazyEnvironment('AVATAO_', 'taoenvtuple').environment diff --git a/lib/envvars/__init__.py b/tfw/config/lazy_environment.py similarity index 100% rename from lib/envvars/__init__.py rename to tfw/config/lazy_environment.py diff --git a/lib/tfw/event_handlers.py b/tfw/event_handlers.py similarity index 100% rename from lib/tfw/event_handlers.py rename to tfw/event_handlers.py diff --git a/lib/tfw/fsm/__init__.py b/tfw/fsm/__init__.py similarity index 100% rename from lib/tfw/fsm/__init__.py rename to tfw/fsm/__init__.py diff --git a/lib/tfw/fsm/fsm_base.py b/tfw/fsm/fsm_base.py similarity index 100% rename from lib/tfw/fsm/fsm_base.py rename to tfw/fsm/fsm_base.py diff --git a/lib/tfw/fsm/linear_fsm.py b/tfw/fsm/linear_fsm.py similarity index 100% rename from lib/tfw/fsm/linear_fsm.py rename to tfw/fsm/linear_fsm.py diff --git a/lib/tfw/fsm/yaml_fsm.py b/tfw/fsm/yaml_fsm.py similarity index 100% rename from lib/tfw/fsm/yaml_fsm.py rename to tfw/fsm/yaml_fsm.py diff --git a/lib/tfw/components/__init__.py b/tfw/internals/__init__.py similarity index 100% rename from lib/tfw/components/__init__.py rename to tfw/internals/__init__.py diff --git a/lib/tfw/internals/callback_mixin.py b/tfw/internals/callback_mixin.py similarity index 100% rename from lib/tfw/internals/callback_mixin.py rename to tfw/internals/callback_mixin.py diff --git a/lib/tfw/internals/crypto.py b/tfw/internals/crypto.py similarity index 100% rename from lib/tfw/internals/crypto.py rename to tfw/internals/crypto.py diff --git a/lib/tfw/internals/event_handling/__init__.py b/tfw/internals/event_handling/__init__.py similarity index 100% rename from lib/tfw/internals/event_handling/__init__.py rename to tfw/internals/event_handling/__init__.py diff --git a/lib/tfw/internals/event_handling/event_handler.py b/tfw/internals/event_handling/event_handler.py similarity index 100% rename from lib/tfw/internals/event_handling/event_handler.py rename to tfw/internals/event_handling/event_handler.py diff --git a/lib/tfw/internals/event_handling/event_handler_factory_base.py b/tfw/internals/event_handling/event_handler_factory_base.py similarity index 100% rename from lib/tfw/internals/event_handling/event_handler_factory_base.py rename to tfw/internals/event_handling/event_handler_factory_base.py diff --git a/lib/tfw/internals/event_handling/fsm_aware.py b/tfw/internals/event_handling/fsm_aware.py similarity index 100% rename from lib/tfw/internals/event_handling/fsm_aware.py rename to tfw/internals/event_handling/fsm_aware.py diff --git a/lib/tfw/internals/event_handling/fsm_aware_event_handler.py b/tfw/internals/event_handling/fsm_aware_event_handler.py similarity index 100% rename from lib/tfw/internals/event_handling/fsm_aware_event_handler.py rename to tfw/internals/event_handling/fsm_aware_event_handler.py diff --git a/lib/tfw/internals/event_handling/test_event_handler.py b/tfw/internals/event_handling/test_event_handler.py similarity index 100% rename from lib/tfw/internals/event_handling/test_event_handler.py rename to tfw/internals/event_handling/test_event_handler.py diff --git a/lib/tfw/internals/inotify/__init__.py b/tfw/internals/inotify/__init__.py similarity index 100% rename from lib/tfw/internals/inotify/__init__.py rename to tfw/internals/inotify/__init__.py diff --git a/lib/tfw/internals/inotify/inotify.py b/tfw/internals/inotify/inotify.py similarity index 100% rename from lib/tfw/internals/inotify/inotify.py rename to tfw/internals/inotify/inotify.py diff --git a/lib/tfw/internals/inotify/test_inotify.py b/tfw/internals/inotify/test_inotify.py similarity index 100% rename from lib/tfw/internals/inotify/test_inotify.py rename to tfw/internals/inotify/test_inotify.py diff --git a/lib/tfw/internals/lazy.py b/tfw/internals/lazy.py similarity index 100% rename from lib/tfw/internals/lazy.py rename to tfw/internals/lazy.py diff --git a/lib/tfw/internals/networking/__init__.py b/tfw/internals/networking/__init__.py similarity index 100% rename from lib/tfw/internals/networking/__init__.py rename to tfw/internals/networking/__init__.py diff --git a/lib/tfw/internals/networking/event_handler_connector.py b/tfw/internals/networking/event_handler_connector.py similarity index 100% rename from lib/tfw/internals/networking/event_handler_connector.py rename to tfw/internals/networking/event_handler_connector.py diff --git a/lib/tfw/internals/networking/scope.py b/tfw/internals/networking/scope.py similarity index 100% rename from lib/tfw/internals/networking/scope.py rename to tfw/internals/networking/scope.py diff --git a/lib/tfw/internals/networking/serialization.py b/tfw/internals/networking/serialization.py similarity index 100% rename from lib/tfw/internals/networking/serialization.py rename to tfw/internals/networking/serialization.py diff --git a/lib/tfw/internals/networking/server_connector.py b/tfw/internals/networking/server_connector.py similarity index 100% rename from lib/tfw/internals/networking/server_connector.py rename to tfw/internals/networking/server_connector.py diff --git a/lib/tfw/internals/server/__init__.py b/tfw/internals/server/__init__.py similarity index 100% rename from lib/tfw/internals/server/__init__.py rename to tfw/internals/server/__init__.py diff --git a/lib/tfw/internals/server/zmq_websocket_router.py b/tfw/internals/server/zmq_websocket_router.py similarity index 100% rename from lib/tfw/internals/server/zmq_websocket_router.py rename to tfw/internals/server/zmq_websocket_router.py diff --git a/lib/tfw/logging.py b/tfw/logging.py similarity index 100% rename from lib/tfw/logging.py rename to tfw/logging.py diff --git a/lib/tfw/main/__init__.py b/tfw/main/__init__.py similarity index 100% rename from lib/tfw/main/__init__.py rename to tfw/main/__init__.py diff --git a/lib/tfw/main/event_handler_factory.py b/tfw/main/event_handler_factory.py similarity index 100% rename from lib/tfw/main/event_handler_factory.py rename to tfw/main/event_handler_factory.py diff --git a/lib/tfw/main/signal_handling.py b/tfw/main/signal_handling.py similarity index 100% rename from lib/tfw/main/signal_handling.py rename to tfw/main/signal_handling.py diff --git a/lib/tfw/main/tfw_connector.py b/tfw/main/tfw_connector.py similarity index 100% rename from lib/tfw/main/tfw_connector.py rename to tfw/main/tfw_connector.py diff --git a/lib/tfw/main/tfw_server.py b/tfw/main/tfw_server.py similarity index 100% rename from lib/tfw/main/tfw_server.py rename to tfw/main/tfw_server.py From f2b6123d4178cb5c9999fbd0a5294699210183b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 24 Jul 2019 16:32:57 +0200 Subject: [PATCH 090/174] Remove unnecessary newlines --- tfw/components/fsm/fsm_handler.py | 1 - tfw/components/ide/file_manager/test_file_manager.py | 2 +- tfw/components/ide/ide_handler.py | 1 - tfw/components/pipe_io/pipe_io_handler.py | 1 - tfw/components/process_management/process_handler.py | 1 - tfw/components/process_management/process_log_handler.py | 1 - tfw/components/snapshots/snapshot_handler.py | 1 - tfw/components/terminal/terminal_handler.py | 1 - tfw/config/envvars.py | 1 - tfw/main/tfw_server.py | 1 - 10 files changed, 1 insertion(+), 10 deletions(-) diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index 2dc163e..e86dee2 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -5,7 +5,6 @@ from tfw.internals.networking import Scope from .fsm_updater import FSMUpdater - LOG = logging.getLogger(__name__) diff --git a/tfw/components/ide/file_manager/test_file_manager.py b/tfw/components/ide/file_manager/test_file_manager.py index 380da0a..8f978d3 100644 --- a/tfw/components/ide/file_manager/test_file_manager.py +++ b/tfw/components/ide/file_manager/test_file_manager.py @@ -1,5 +1,4 @@ # pylint: disable=redefined-outer-name - from dataclasses import dataclass from secrets import token_urlsafe from os.path import join @@ -11,6 +10,7 @@ import pytest from .file_manager import FileManager + @dataclass class ManagerContext: folder: str diff --git a/tfw/components/ide/ide_handler.py b/tfw/components/ide/ide_handler.py index 225b80c..7f09247 100644 --- a/tfw/components/ide/ide_handler.py +++ b/tfw/components/ide/ide_handler.py @@ -5,7 +5,6 @@ from tfw.internals.inotify import InotifyObserver from .file_manager import FileManager - LOG = logging.getLogger(__name__) BUILD_ARTIFACTS = ( diff --git a/tfw/components/pipe_io/pipe_io_handler.py b/tfw/components/pipe_io/pipe_io_handler.py index 4f6a850..115b508 100644 --- a/tfw/components/pipe_io/pipe_io_handler.py +++ b/tfw/components/pipe_io/pipe_io_handler.py @@ -12,7 +12,6 @@ from contextlib import suppress from .pipe_io_server import PipeIOServer, terminate_process_on_failure - LOG = logging.getLogger(__name__) DEFAULT_PERMISSIONS = 0o600 diff --git a/tfw/components/process_management/process_handler.py b/tfw/components/process_management/process_handler.py index 54928fd..becd28d 100644 --- a/tfw/components/process_management/process_handler.py +++ b/tfw/components/process_management/process_handler.py @@ -5,7 +5,6 @@ from tfw.internals.networking import Scope from .supervisor import ProcessManager, ProcessLogManager - LOG = logging.getLogger(__name__) diff --git a/tfw/components/process_management/process_log_handler.py b/tfw/components/process_management/process_log_handler.py index 4ee084a..a7ed020 100644 --- a/tfw/components/process_management/process_log_handler.py +++ b/tfw/components/process_management/process_log_handler.py @@ -2,7 +2,6 @@ import logging from .log_inotify_observer import LogInotifyObserver - LOG = logging.getLogger(__name__) diff --git a/tfw/components/snapshots/snapshot_handler.py b/tfw/components/snapshots/snapshot_handler.py index 2ca9d28..c60c720 100644 --- a/tfw/components/snapshots/snapshot_handler.py +++ b/tfw/components/snapshots/snapshot_handler.py @@ -10,7 +10,6 @@ from tfw.internals.networking import Scope from .snapshot_provider import SnapshotProvider - LOG = logging.getLogger(__name__) diff --git a/tfw/components/terminal/terminal_handler.py b/tfw/components/terminal/terminal_handler.py index 2cbdff4..a0a5f0c 100644 --- a/tfw/components/terminal/terminal_handler.py +++ b/tfw/components/terminal/terminal_handler.py @@ -3,7 +3,6 @@ import logging from .history_monitor import BashMonitor from .terminado_mini_server import TerminadoMiniServer - LOG = logging.getLogger(__name__) diff --git a/tfw/config/envvars.py b/tfw/config/envvars.py index 1e4c89a..dcf767d 100644 --- a/tfw/config/envvars.py +++ b/tfw/config/envvars.py @@ -1,5 +1,4 @@ from .lazy_environment import LazyEnvironment - TFWENV = LazyEnvironment('TFW_', 'tfwenvtuple').environment TAOENV = LazyEnvironment('AVATAO_', 'taoenvtuple').environment diff --git a/tfw/main/tfw_server.py b/tfw/main/tfw_server.py index 22d216f..4cc1c5e 100644 --- a/tfw/main/tfw_server.py +++ b/tfw/main/tfw_server.py @@ -6,7 +6,6 @@ from tfw.internals.networking import EventHandlerConnector from tfw.internals.server import ZMQWebSocketRouter from tfw.config import TFWENV - LOG = logging.getLogger(__name__) From 42f959878f14727bf859d4d4ac0ee6b6f413f684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 24 Jul 2019 16:47:03 +0200 Subject: [PATCH 091/174] Add signal handling to tfw_server.py --- supervisor/tfw_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/supervisor/tfw_server.py b/supervisor/tfw_server.py index f3ad4be..eccd753 100644 --- a/supervisor/tfw_server.py +++ b/supervisor/tfw_server.py @@ -2,7 +2,7 @@ from sys import stderr from tornado.ioloop import IOLoop -from tfw.main import TFWServer +from tfw.main import TFWServer, setup_signal_handlers from tfw.config import TFWENV from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter @@ -13,4 +13,6 @@ if __name__ == '__main__': Log(TFWENV.LOGFILE, VerboseLogFormatter()) ]).start() TFWServer().listen() + + setup_signal_handlers() IOLoop.instance().start() From 3fca5552519db222ec82e5639ac672ad884631eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 24 Jul 2019 17:22:08 +0200 Subject: [PATCH 092/174] Move TFWRouter to separate file --- tfw/internals/server/tfw_router.py | 22 +++++++++++++++++++ tfw/internals/server/zmq_websocket_router.py | 23 +------------------- 2 files changed, 23 insertions(+), 22 deletions(-) create mode 100644 tfw/internals/server/tfw_router.py diff --git a/tfw/internals/server/tfw_router.py b/tfw/internals/server/tfw_router.py new file mode 100644 index 0000000..c2a18ab --- /dev/null +++ b/tfw/internals/server/tfw_router.py @@ -0,0 +1,22 @@ +from tfw.internals.networking import Scope + + +class TFWRouter: + def __init__(self, send_to_zmq, send_to_websockets): + self.send_to_zmq = send_to_zmq + self.send_to_websockets = send_to_websockets + + def route(self, message): + scope = Scope(message.pop('scope', 'zmq')) + + routing_table = { + Scope.ZMQ: self.send_to_zmq, + Scope.WEBSOCKET: self.send_to_websockets, + Scope.BROADCAST: self.broadcast + } + action = routing_table[scope] + action(message) + + def broadcast(self, message): + self.send_to_zmq(message) + self.send_to_websockets(message) diff --git a/tfw/internals/server/zmq_websocket_router.py b/tfw/internals/server/zmq_websocket_router.py index cdafc2f..3f5e7fc 100644 --- a/tfw/internals/server/zmq_websocket_router.py +++ b/tfw/internals/server/zmq_websocket_router.py @@ -3,7 +3,7 @@ import logging from tornado.websocket import WebSocketHandler -from tfw.internals.networking import Scope +from .tfw_router import TFWRouter LOG = logging.getLogger(__name__) @@ -46,24 +46,3 @@ class ZMQWebSocketRouter(WebSocketHandler): # much secure, very cors, wow def check_origin(self, origin): return True - - -class TFWRouter: - def __init__(self, send_to_zmq, send_to_websockets): - self.send_to_zmq = send_to_zmq - self.send_to_websockets = send_to_websockets - - def route(self, message): - scope = Scope(message.pop('scope', 'zmq')) - - routing_table = { - Scope.ZMQ: self.send_to_zmq, - Scope.WEBSOCKET: self.send_to_websockets, - Scope.BROADCAST: self.broadcast - } - action = routing_table[scope] - action(message) - - def broadcast(self, message): - self.send_to_zmq(message) - self.send_to_websockets(message) From 9a4da0b36f3e8e1bbca2213f553cc47250288a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 29 Jul 2019 13:02:46 +0200 Subject: [PATCH 093/174] Implement bootiful unit tests for TFW network stack --- pytest.ini | 2 + tfw/internals/networking/test_networking.py | 106 ++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 pytest.ini create mode 100644 tfw/internals/networking/test_networking.py diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..72a218a --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +filterwarnings = ignore::DeprecationWarning:zmq diff --git a/tfw/internals/networking/test_networking.py b/tfw/internals/networking/test_networking.py new file mode 100644 index 0000000..cc23069 --- /dev/null +++ b/tfw/internals/networking/test_networking.py @@ -0,0 +1,106 @@ +# pylint: disable=redefined-outer-name +from os.path import join +from secrets import token_urlsafe +from random import randint +from tempfile import TemporaryDirectory + +import pytest +from tornado.ioloop import IOLoop + +from tfw.internals.networking import EventHandlerConnector, ServerConnector + + +@pytest.fixture +def _connectors(): + with TemporaryDirectory() as tmpdir: + down_sock = join(tmpdir, 'down') + up_sock = join(tmpdir, 'up') + server_downlink = f'ipc://{down_sock}' + server_uplink = f'ipc://{up_sock}' + + ec = EventHandlerConnector(server_downlink, server_uplink) + sc = ServerConnector(server_uplink, server_downlink) + yield ec, sc + sc.close() + ec.close() + + +@pytest.fixture +def eh_connector(_connectors): + eh_connector, _ = _connectors + yield eh_connector + + +@pytest.fixture +def server_connector(_connectors): + _, server_connector = _connectors + yield server_connector + + +def run_ioloop_once(): + # hack: we have to wait for the messages to get through + # the network stack of the OS while the IOLoop is waiting + # for them via select/epoll/kqueue + IOLoop.current().call_later(0.1, IOLoop.current().stop) + IOLoop.current().start() + + +@pytest.fixture +def test_messages(): + random_str = lambda: token_urlsafe(randint(4, 8)) + yield [ + { + 'key': random_str(), + random_str(): randint(8192, 16384), + random_str(): random_str(), + random_str(): { + random_str(): random_str(), + random_str(): {random_str(): random_str()} + }, + random_str(): [random_str(), random_str()] + } + for _ in range(randint(8, 16)) + ] + + +def test_server_downlink(eh_connector, server_connector, test_messages): + messages = [] + eh_connector.register_callback(messages.append) + + for message in test_messages: + server_connector.send_message(message) + + run_ioloop_once() + + assert messages == test_messages + + +def test_server_uplink(eh_connector, server_connector, test_messages): + messages = [] + server_connector.subscribe('') + server_connector.register_callback(messages.append) + + for message in test_messages: + eh_connector.send_message(message) + + run_ioloop_once() + + assert messages == test_messages + + +def test_connector_downlink_subscribe(eh_connector, server_connector): + key1_messages = [{'key': '1', 'data': i} for i in range(randint(128, 256))] + key2_messages = [{'key': '2', 'data': i} for i in range(randint(128, 256))] + all_messages = key1_messages + key2_messages + + messages = [] + server_connector.subscribe('1') + server_connector.register_callback(messages.append) + + for message in all_messages: + eh_connector.send_message(message) + + run_ioloop_once() + + assert messages == key1_messages + assert all((msg not in messages for msg in key2_messages)) From fd92c443b2c7f33fff36bd2fcc4f7b134be28765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 29 Jul 2019 18:31:01 +0200 Subject: [PATCH 094/174] Update PipeIOServer dependency --- .../pipe_io/pipe_io_server/__init__.py | 2 + tfw/components/pipe_io/pipe_io_server/pipe.py | 2 + .../pipe_io/pipe_io_server/pipe_io_server.py | 92 +++------ .../pipe_io/pipe_io_server/pipe_io_thread.py | 45 +++++ .../pipe_io_server/pipe_reader_server.py | 39 ++++ .../pipe_io_server/pipe_writer_server.py | 38 ++++ .../pipe_io_server/test_pipe_io_server.py | 187 ++++++++++++++++++ 7 files changed, 335 insertions(+), 70 deletions(-) create mode 100644 tfw/components/pipe_io/pipe_io_server/pipe_io_thread.py create mode 100644 tfw/components/pipe_io/pipe_io_server/pipe_reader_server.py create mode 100644 tfw/components/pipe_io/pipe_io_server/pipe_writer_server.py create mode 100644 tfw/components/pipe_io/pipe_io_server/test_pipe_io_server.py diff --git a/tfw/components/pipe_io/pipe_io_server/__init__.py b/tfw/components/pipe_io/pipe_io_server/__init__.py index cab334b..b0d450a 100644 --- a/tfw/components/pipe_io/pipe_io_server/__init__.py +++ b/tfw/components/pipe_io/pipe_io_server/__init__.py @@ -1,2 +1,4 @@ from .pipe_io_server import PipeIOServer +from .pipe_reader_server import PipeReaderServer +from .pipe_writer_server import PipeWriterServer from .terminate_process_on_failure import terminate_process_on_failure diff --git a/tfw/components/pipe_io/pipe_io_server/pipe.py b/tfw/components/pipe_io/pipe_io_server/pipe.py index eb021ae..f83664a 100644 --- a/tfw/components/pipe_io/pipe_io_server/pipe.py +++ b/tfw/components/pipe_io/pipe_io_server/pipe.py @@ -1,6 +1,8 @@ from os import mkfifo, remove, chmod from os.path import exists +DEFAULT_PERMISSIONS = 0o600 + class Pipe: def __init__(self, path): diff --git a/tfw/components/pipe_io/pipe_io_server/pipe_io_server.py b/tfw/components/pipe_io/pipe_io_server/pipe_io_server.py index 2715f40..8204adb 100644 --- a/tfw/components/pipe_io/pipe_io_server/pipe_io_server.py +++ b/tfw/components/pipe_io/pipe_io_server/pipe_io_server.py @@ -1,73 +1,25 @@ -from abc import ABC, abstractmethod -from threading import Thread, Event -from typing import Callable - -from .pipe_reader_thread import PipeReaderThread -from .pipe_writer_thread import PipeWriterThread -from .pipe import Pipe -from .terminate_process_on_failure import terminate_process_on_failure +from .pipe import DEFAULT_PERMISSIONS +from .pipe_reader_server import PipeReaderServer +from .pipe_writer_server import PipeWriterServer -class PipeIOServer(ABC, Thread): - def __init__(self, in_pipe=None, out_pipe=None, permissions=0o600): - super().__init__(daemon=True) - self._in_pipe, self._out_pipe = in_pipe, out_pipe - self._create_pipes(permissions) - self._stop_event = Event() - self._reader_thread, self._writer_thread = self._create_io_threads() - self._io_threads = (self._reader_thread, self._writer_thread) - self._on_stop = lambda: None +class PipeIOServer(PipeReaderServer, PipeWriterServer): + # pylint: disable=abstract-method + def __init__( + self, + in_pipe, + out_pipe, + permissions=DEFAULT_PERMISSIONS, + manage_pipes=True + ): + super().__init__( + in_pipe=in_pipe, + out_pipe=out_pipe, + permissions=permissions, + manage_pipes=manage_pipes + ) - def _create_pipes(self, permissions): - Pipe(self.in_pipe).recreate(permissions) - Pipe(self.out_pipe).recreate(permissions) - - @property - def in_pipe(self): - return self._in_pipe - - @property - def out_pipe(self): - return self._out_pipe - - def _create_io_threads(self): - reader_thread = PipeReaderThread(self.in_pipe, self._stop_event, self.handle_message) - writer_thread = PipeWriterThread(self.out_pipe, self._stop_event) - return reader_thread, writer_thread - - @abstractmethod - def handle_message(self, message): - raise NotImplementedError() - - def send_message(self, message): - self._writer_thread.write(message) - - @terminate_process_on_failure - def run(self): - for thread in self._io_threads: - thread.start() - self._stop_event.wait() - self._stop_threads() - - def stop(self): - self._stop_event.set() - if self.is_alive(): - self.join() - - def _stop_threads(self): - for thread in self._io_threads: - if thread.is_alive(): - thread.stop() - Pipe(self.in_pipe).remove() - Pipe(self.out_pipe).remove() - self._on_stop() - - def _set_on_stop(self, value): - if not isinstance(value, Callable): - raise ValueError("Supplied object is not callable!") - self._on_stop = value - - on_stop = property(fset=_set_on_stop) - - def wait(self): - self._stop_event.wait() + def _io_threads(self): + # pylint: disable=no-member + yield from PipeReaderServer._io_threads(self) + yield from PipeWriterServer._io_threads(self) diff --git a/tfw/components/pipe_io/pipe_io_server/pipe_io_thread.py b/tfw/components/pipe_io/pipe_io_server/pipe_io_thread.py new file mode 100644 index 0000000..91a50b4 --- /dev/null +++ b/tfw/components/pipe_io/pipe_io_server/pipe_io_thread.py @@ -0,0 +1,45 @@ +from threading import Thread, Event + +from .terminate_process_on_failure import terminate_process_on_failure + + +class PipeIOThread(Thread): + def __init__(self): + super().__init__(daemon=True) + self._stop_event = Event() + self.__io_threads = [] + + @terminate_process_on_failure + def run(self): + self.__io_threads.extend(self._io_threads()) + for thread in self.__io_threads: + thread.start() + self._stop_event.wait() + self._stop_threads() + + def _io_threads(self): + raise NotImplementedError() + + def _stop_threads(self): + for thread in self.__io_threads: + if thread.is_alive(): + thread.stop() + self.on_stop() + + def on_stop(self): + pass + + def stop(self): + self._stop_event.set() + if self.is_alive(): + self.join() + + def wait(self): + self._stop_event.wait() + + def __enter__(self): + self.start() + return self + + def __exit__(self, type_, value, tb): + self.stop() diff --git a/tfw/components/pipe_io/pipe_io_server/pipe_reader_server.py b/tfw/components/pipe_io/pipe_io_server/pipe_reader_server.py new file mode 100644 index 0000000..e26bb87 --- /dev/null +++ b/tfw/components/pipe_io/pipe_io_server/pipe_reader_server.py @@ -0,0 +1,39 @@ +from .pipe import Pipe, DEFAULT_PERMISSIONS +from .pipe_io_thread import PipeIOThread +from .pipe_reader_thread import PipeReaderThread + + +class PipeReaderServer(PipeIOThread): + def __init__( + self, + in_pipe, + permissions=DEFAULT_PERMISSIONS, + manage_pipes=True, + **kwargs + ): + super().__init__(**kwargs) + self._reader_thread = None + self._manage_pipes = manage_pipes + self._in_pipe = in_pipe + if self._manage_pipes: + Pipe(self.in_pipe).recreate(permissions) + + @property + def in_pipe(self): + return self._in_pipe + + def _io_threads(self): + self._reader_thread = PipeReaderThread( + self.in_pipe, + self._stop_event, + self.handle_message + ) + yield self._reader_thread + + def handle_message(self, message): + raise NotImplementedError() + + def stop(self): + super().stop() + if self._manage_pipes: + Pipe(self.in_pipe).remove() diff --git a/tfw/components/pipe_io/pipe_io_server/pipe_writer_server.py b/tfw/components/pipe_io/pipe_io_server/pipe_writer_server.py new file mode 100644 index 0000000..1161917 --- /dev/null +++ b/tfw/components/pipe_io/pipe_io_server/pipe_writer_server.py @@ -0,0 +1,38 @@ +from .pipe import Pipe, DEFAULT_PERMISSIONS +from .pipe_io_thread import PipeIOThread +from .pipe_writer_thread import PipeWriterThread + + +class PipeWriterServer(PipeIOThread): + def __init__( + self, + out_pipe, + permissions=DEFAULT_PERMISSIONS, + manage_pipes=True, + **kwargs + ): + super().__init__(**kwargs) + self._writer_thread = None + self._manage_pipes = manage_pipes + self._out_pipe = out_pipe + if self._manage_pipes: + Pipe(self.out_pipe).recreate(permissions) + + @property + def out_pipe(self): + return self._out_pipe + + def _io_threads(self): + self._writer_thread = PipeWriterThread( + self.out_pipe, + self._stop_event + ) + yield self._writer_thread + + def send_message(self, message): + self._writer_thread.write(message) + + def stop(self): + super().stop() + if self._manage_pipes: + Pipe(self.out_pipe).remove() diff --git a/tfw/components/pipe_io/pipe_io_server/test_pipe_io_server.py b/tfw/components/pipe_io/pipe_io_server/test_pipe_io_server.py new file mode 100644 index 0000000..6a08847 --- /dev/null +++ b/tfw/components/pipe_io/pipe_io_server/test_pipe_io_server.py @@ -0,0 +1,187 @@ +# pylint: disable=redefined-outer-name +from os import stat, urandom +from os.path import exists, dirname, realpath, join +from stat import S_ISFIFO +from secrets import token_urlsafe +from random import randint, getrandbits, uniform +from threading import Thread +from json import dumps, loads + +import pytest + +from .pipe_io_server import PipeIOServer + + +class EchoPipeIOServer(PipeIOServer): + def handle_message(self, message): + self.send_message(message) + + +@pytest.fixture +def io_pipes(): + with EchoPipeIOServer(*get_test_init_params()) as pipe_io: + with IOPipes(pipe_io.in_pipe, pipe_io.out_pipe) as io_pipes: + yield io_pipes + + +def get_test_init_params(): + here = dirname(realpath(__file__)) + return join(here, 'in_pipe_tests'), join(here, 'out_pipe_tests') + + +def raise_if_thread_blocks(*, target, unblock_function): + thread = Thread(target=target) + thread.start() + unblock_function() + thread.join(timeout=1) + if thread.is_alive(): + raise RuntimeError('PipeIOServer failed to shut down!') + + +class IOPipes: + def __init__(self, in_pipe_path, out_pipe_path): + self.in_pipe_path = in_pipe_path + self.out_pipe_path = out_pipe_path + + def __enter__(self): + # pylint: disable=attribute-defined-outside-init + self.in_pipe = open(self.in_pipe_path, 'wb') + self.out_pipe = open(self.out_pipe_path, 'rb') + return self + + def __exit__(self, type_, value, traceback): + self.close() + + def close(self): + self.in_pipe.close() + self.out_pipe.close() + + def send_message(self, message): + self.in_pipe.write(message + b'\n') + self.in_pipe.flush() + + def recv(self): + return self.out_pipe.readline().rstrip(b'\n') + + +def pipes_exist(*paths): + predicate = lambda path: exists(path) and S_ISFIFO(stat(path).st_mode) + return all(predicate(path) for path in paths) + + +def test_manage_pipes(): + pipe_io = PipeIOServer(*get_test_init_params(), manage_pipes=True) + assert pipes_exist(pipe_io.in_pipe, pipe_io.out_pipe) + pipe_io.stop() + assert not pipes_exist(pipe_io.in_pipe, pipe_io.out_pipe) + + +def test_no_manage_pipes(): + pipe_io = PipeIOServer(*get_test_init_params(), manage_pipes=False) + assert not pipes_exist(pipe_io.in_pipe, pipe_io.out_pipe) + pipe_io.stop() + assert not pipes_exist(pipe_io.in_pipe, pipe_io.out_pipe) + + +def test_stop(): + pipe_io = EchoPipeIOServer(*get_test_init_params()) + pipe_io.start() + raise_if_thread_blocks(target=pipe_io.wait, unblock_function=pipe_io.stop) + + pipe_io = EchoPipeIOServer(*get_test_init_params()) + pipe_io.start() + with IOPipes(pipe_io.in_pipe, pipe_io.out_pipe) as iopipes: + raise_if_thread_blocks(target=pipe_io.wait, unblock_function=pipe_io.stop) + + pipe_io = EchoPipeIOServer(*get_test_init_params()) + pipe_io.start() + with IOPipes(pipe_io.in_pipe, pipe_io.out_pipe) as iopipes: + test_message = token_urlsafe(randint(128, 256)) + iopipes.send_message(test_message.encode()) + assert test_message == iopipes.recv().decode() + iopipes.send_message(test_message.encode()) + raise_if_thread_blocks(target=pipe_io.wait, unblock_function=pipe_io.stop) + + +@pytest.mark.parametrize( + 'test_data', [ + 'Cats and cheese', + 'You ever wonder why we are here?', + 'Lorem ipsum dolor sit amet', + 'You always have a plan, Dutch!', + ] +) +def test_io(io_pipes, test_data): + io_pipes.send_message(test_data.encode()) + assert io_pipes.recv().decode() == test_data + + +def test_io_random(io_pipes): + test_data = token_urlsafe(512) + for _ in range(100): + io_pipes.send_message(test_data.encode()) + assert io_pipes.recv().decode() == test_data + +@pytest.mark.parametrize( + 'test_data_size', [ + 1024, + 1024*1024, + 2*1024*1024, + 4*1024*1024, + 8*1024*1024, + 16*1024*1024, + 32*1024*1024 + ] +) +def test_io_large_data(io_pipes, test_data_size): + test_data = urandom(test_data_size).replace(b'\n', b'') + io_pipes.send_message(test_data) + received_data = io_pipes.recv() + assert received_data == test_data + + +def test_io_stress(io_pipes): + for _ in range(2222): + test_data = urandom(randint(1, 1024)).replace(b'\n', b'') + io_pipes.send_message(test_data) + assert io_pipes.recv() == test_data + + +def test_io_newlines(io_pipes): + times = randint(1, 512) + io_pipes.send_message(b'\n' * times) + for _ in range(times + 1): # IOPipes.send appends +1 + assert io_pipes.recv() == b'' + + +def test_json_io(io_pipes): + for _ in range(10): + test_data = { + f'{token_urlsafe(8)}': randint(1, 2 ** 20), + f'{token_urlsafe(9)}': [randint(1, 2 **10) for i in range(10)], + f'{token_urlsafe(4)}': f'{token_urlsafe(8)}\\\n{token_urlsafe(8)}\n{randint(1, 2 ** 10)}', + f'{token_urlsafe(11)}': { + f'{token_urlsafe(8)}': '', + f'{token_urlsafe(3)}': f'{token_urlsafe(8)}\n{token_urlsafe(8)}\n\\n{token_urlsafe(8)}', + f'{token_urlsafe(44)}': f'{token_urlsafe(8)}\n{token_urlsafe(8)} {token_urlsafe(8)}', + f'{token_urlsafe(6)}\n{token_urlsafe(4)}': bool(getrandbits(1)), + f'{token_urlsafe(8)}': None, + f'{token_urlsafe(21)} {token_urlsafe(4)}': None, + f'{token_urlsafe(3)}': uniform(randint(1, 100), randint(1, 100)), + f'{token_urlsafe(8)}': [token_urlsafe(4) for i in range(10)], + } + } + io_pipes.send_message(dumps(test_data).encode()) + assert loads(io_pipes.recv()) == test_data + + +def test_assign_message_handler(): + pipe_io = PipeIOServer(*get_test_init_params()) + pipe_io.handle_message = lambda msg: pipe_io.send_message(msg * 2) + pipe_io.start() + with IOPipes(pipe_io.in_pipe, pipe_io.out_pipe) as io_pipes: + for _ in range(100): + test_data = token_urlsafe(32).encode() + io_pipes.send_message(test_data) + assert io_pipes.recv() == test_data * 2 + pipe_io.stop() From 30223fe99ff410a0eaa2d2824cc5eeddbe1d4a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 30 Jul 2019 15:17:29 +0200 Subject: [PATCH 095/174] =?UTF-8?q?Rename=20ServerConnector=20interface=20?= =?UTF-8?q?to=20Connector=20=C2=AF\=5F(=E3=83=84)=5F/=C2=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tfw/components/frontend/frontend_handler.py | 4 +- tfw/components/fsm/fsm_handler.py | 4 +- tfw/components/ide/ide_handler.py | 4 +- tfw/components/pipe_io/pipe_io_handler.py | 6 +-- .../log_inotify_observer.py | 6 +-- .../process_management/process_handler.py | 4 +- .../process_management/process_log_handler.py | 4 +- tfw/components/snapshots/snapshot_handler.py | 4 +- tfw/components/terminal/terminal_handler.py | 4 +- tfw/internals/event_handling/event_handler.py | 12 ++--- .../event_handler_factory_base.py | 14 +++--- .../event_handling/fsm_aware_event_handler.py | 6 +-- .../event_handling/test_event_handler.py | 48 +++++++++---------- tfw/internals/networking/__init__.py | 4 +- tfw/internals/networking/test_networking.py | 48 +++++++++---------- .../{server_connector.py => zmq_connector.py} | 14 +++--- ...t_handler_connector.py => zmq_listener.py} | 14 +++--- tfw/internals/server/zmq_websocket_router.py | 6 +-- tfw/main/event_handler_factory.py | 2 +- tfw/main/tfw_connector.py | 6 +-- tfw/main/tfw_server.py | 6 +-- 21 files changed, 110 insertions(+), 110 deletions(-) rename tfw/internals/networking/{server_connector.py => zmq_connector.py} (83%) rename tfw/internals/networking/{event_handler_connector.py => zmq_listener.py} (75%) diff --git a/tfw/components/frontend/frontend_handler.py b/tfw/components/frontend/frontend_handler.py index b5e7c8b..5829ad7 100644 --- a/tfw/components/frontend/frontend_handler.py +++ b/tfw/components/frontend/frontend_handler.py @@ -7,12 +7,12 @@ class FrontendHandler: keys = ['message', 'queueMessages', 'dashboard', 'console'] def __init__(self): - self.server_connector = None + self.connector = None self.keys = [*type(self).keys, 'recover'] self._frontend_message_storage = FrontendMessageStorage(type(self).keys) def send_message(self, message): - self.server_connector.send_message(message, scope=Scope.WEBSOCKET) + self.connector.send_message(message, scope=Scope.WEBSOCKET) def handle_event(self, message, _): self._frontend_message_storage.save_message(message) diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index e86dee2..18302d2 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -35,14 +35,14 @@ class FSMHandler: 'update': self.handle_update } - def handle_event(self, message, server_connector): + def handle_event(self, message, connector): try: message = self.command_handlers[message['data']['command']](message) if message: fsm_update_message = self._fsm_updater.fsm_update sign_message(self.auth_key, message) sign_message(self.auth_key, fsm_update_message) - server_connector.send_message(fsm_update_message, Scope.BROADCAST) + connector.send_message(fsm_update_message, Scope.BROADCAST) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/tfw/components/ide/ide_handler.py b/tfw/components/ide/ide_handler.py index 7f09247..38f7842 100644 --- a/tfw/components/ide/ide_handler.py +++ b/tfw/components/ide/ide_handler.py @@ -55,7 +55,7 @@ class IdeHandler: :param selected_file: file that is selected by default :param exclude: list of filenames that should not appear between files (for .o, .pyc, etc.) """ - self.server_connector = None + self.connector = None try: self.filemanager = FileManager( allowed_directories=allowed_directories, @@ -90,7 +90,7 @@ class IdeHandler: }) def send_message(self, message): - self.server_connector.send_message(message, scope=Scope.WEBSOCKET) + self.connector.send_message(message, scope=Scope.WEBSOCKET) def read(self, data): """ diff --git a/tfw/components/pipe_io/pipe_io_handler.py b/tfw/components/pipe_io/pipe_io_handler.py index 115b508..74aabe8 100644 --- a/tfw/components/pipe_io/pipe_io_handler.py +++ b/tfw/components/pipe_io/pipe_io_handler.py @@ -20,7 +20,7 @@ class PipeIOHandlerBase: keys = [''] def __init__(self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS): - self.server_connector = None + self.connector = None self.pipe_io = CallbackPipeIOServer( in_pipe_path, out_pipe_path, @@ -56,7 +56,7 @@ class PipeIOHandler(PipeIOHandlerBase): def handle_pipe_event(self, message_bytes): json = loads(message_bytes) - self.server_connector.send_message(json) + self.connector.send_message(json) class TransformerPipeIOHandler(PipeIOHandlerBase): @@ -93,7 +93,7 @@ class TransformerPipeIOHandler(PipeIOHandlerBase): transformed_bytes = self._transform_in(message_bytes) if transformed_bytes: json_message = loads(transformed_bytes) - self.server_connector.send_message(json_message) + self.connector.send_message(json_message) class CommandHandler(PipeIOHandler): diff --git a/tfw/components/process_management/log_inotify_observer.py b/tfw/components/process_management/log_inotify_observer.py index 94400ca..cbbd5dc 100644 --- a/tfw/components/process_management/log_inotify_observer.py +++ b/tfw/components/process_management/log_inotify_observer.py @@ -7,9 +7,9 @@ from .supervisor import ProcessLogManager class LogInotifyObserver(InotifyObserver, ProcessLogManager): - def __init__(self, server_connector, supervisor_uri, process_name, log_tail=0): + def __init__(self, connector, supervisor_uri, process_name, log_tail=0): self._prevent_log_recursion() - self._server_connector = server_connector + self._connector = connector self._process_name = process_name self.log_tail = log_tail self._procinfo = None @@ -35,7 +35,7 @@ class LogInotifyObserver(InotifyObserver, ProcessLogManager): self.paths = self._get_logfiles() def on_modified(self, event): - self._server_connector.send_message({ + self._connector.send_message({ 'key': 'processlog', 'data': { 'command': 'new_log', diff --git a/tfw/components/process_management/process_handler.py b/tfw/components/process_management/process_handler.py index becd28d..1c54118 100644 --- a/tfw/components/process_management/process_handler.py +++ b/tfw/components/process_management/process_handler.py @@ -32,7 +32,7 @@ class ProcessHandler(ProcessManager, ProcessLogManager): 'restart': self.restart_process } - def handle_event(self, message, server_connector): + def handle_event(self, message, connector): try: data = message['data'] try: @@ -48,6 +48,6 @@ class ProcessHandler(ProcessManager, ProcessLogManager): data['process_name'], self.log_tail ) - server_connector.send_message(message, scope=Scope.WEBSOCKET) + connector.send_message(message, scope=Scope.WEBSOCKET) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/tfw/components/process_management/process_log_handler.py b/tfw/components/process_management/process_log_handler.py index a7ed020..8386120 100644 --- a/tfw/components/process_management/process_log_handler.py +++ b/tfw/components/process_management/process_log_handler.py @@ -17,7 +17,7 @@ class ProcessLogHandler: The API of each command is documented in their respective handler. """ def __init__(self, *, process_name, supervisor_uri, log_tail=0): - self.server_connector = None + self.connector = None self.process_name = process_name self._supervisor_uri = supervisor_uri self._initial_log_tail = log_tail @@ -30,7 +30,7 @@ class ProcessLogHandler: def start(self): self._monitor = LogInotifyObserver( - server_connector=self.server_connector, + connector=self.connector, supervisor_uri=self._supervisor_uri, process_name=self.process_name, log_tail=self._initial_log_tail diff --git a/tfw/components/snapshots/snapshot_handler.py b/tfw/components/snapshots/snapshot_handler.py index c60c720..f992b07 100644 --- a/tfw/components/snapshots/snapshot_handler.py +++ b/tfw/components/snapshots/snapshot_handler.py @@ -45,11 +45,11 @@ class SnapshotHandler: makedirs(git_dir, exist_ok=True) return git_dir - def handle_event(self, message, server_connector): + def handle_event(self, message, connector): try: data = message['data'] message['data'] = self.command_handlers[data['command']](data) - server_connector.send_message(message, scope=Scope.WEBSOCKET) + connector.send_message(message, scope=Scope.WEBSOCKET) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/tfw/components/terminal/terminal_handler.py b/tfw/components/terminal/terminal_handler.py index a0a5f0c..c8df229 100644 --- a/tfw/components/terminal/terminal_handler.py +++ b/tfw/components/terminal/terminal_handler.py @@ -22,7 +22,7 @@ class TerminalHandler: :param key: key this EventHandler listens to :param monitor: tfw.components.HistoryMonitor instance to read command history from """ - self.server_connector = None + self.connector = None self._histfile = histfile self._historymonitor = None bash_as_user_cmd = ['sudo', '-u', user, 'bash'] @@ -42,7 +42,7 @@ class TerminalHandler: self.terminado_server.listen() def start(self): - self._historymonitor = BashMonitor(self.server_connector, self._histfile) + self._historymonitor = BashMonitor(self.connector, self._histfile) self._historymonitor.start() @property diff --git a/tfw/internals/event_handling/event_handler.py b/tfw/internals/event_handling/event_handler.py index a75381e..f288042 100644 --- a/tfw/internals/event_handling/event_handler.py +++ b/tfw/internals/event_handling/event_handler.py @@ -1,17 +1,17 @@ class EventHandler: _instances = set() - def __init__(self, server_connector): + def __init__(self, connector): type(self)._instances.add(self) - self.server_connector = server_connector + self.connector = connector def start(self): - self.server_connector.register_callback(self._event_callback) + self.connector.register_callback(self._event_callback) def _event_callback(self, message): - self.handle_event(message, self.server_connector) + self.handle_event(message, self.connector) - def handle_event(self, message, server_connector): + def handle_event(self, message, connector): raise NotImplementedError() @classmethod @@ -20,7 +20,7 @@ class EventHandler: instance.stop() def stop(self): - self.server_connector.close() + self.connector.close() self.cleanup() def cleanup(self): diff --git a/tfw/internals/event_handling/event_handler_factory_base.py b/tfw/internals/event_handling/event_handler_factory_base.py index 4c4adcd..b62d9f1 100644 --- a/tfw/internals/event_handling/event_handler_factory_base.py +++ b/tfw/internals/event_handling/event_handler_factory_base.py @@ -6,15 +6,15 @@ from .event_handler import EventHandler class EventHandlerFactoryBase: def build(self, handler_stub, *, keys=None, event_handler_type=EventHandler): builder = EventHandlerBuilder(handler_stub, keys, event_handler_type) - server_connector = self._build_server_connector() - event_handler = builder.build(server_connector) - handler_stub.server_connector = server_connector + connector = self._build_connector() + event_handler = builder.build(connector) + handler_stub.connector = connector with suppress(AttributeError): handler_stub.start() event_handler.start() return event_handler - def _build_server_connector(self): + def _build_connector(self): raise NotImplementedError() @@ -23,9 +23,9 @@ class EventHandlerBuilder: self._analyzer = HandlerStubAnalyzer(event_handler, supplied_keys) self._event_handler_type = event_handler_type - def build(self, server_connector): - event_handler = self._event_handler_type(server_connector) - server_connector.subscribe(*self._try_get_keys(event_handler)) + def build(self, connector): + event_handler = self._event_handler_type(connector) + connector.subscribe(*self._try_get_keys(event_handler)) event_handler.handle_event = self._analyzer.handle_event with suppress(AttributeError): event_handler.cleanup = self._analyzer.cleanup diff --git a/tfw/internals/event_handling/fsm_aware_event_handler.py b/tfw/internals/event_handling/fsm_aware_event_handler.py index 966d4d4..a128de8 100644 --- a/tfw/internals/event_handling/fsm_aware_event_handler.py +++ b/tfw/internals/event_handling/fsm_aware_event_handler.py @@ -8,12 +8,12 @@ class FSMAwareEventHandler(EventHandler, FSMAware): Abstract base class for EventHandlers which automatically keep track of the state of the TFW FSM. """ - def __init__(self, server_connector): - EventHandler.__init__(self, server_connector) + def __init__(self, connector): + EventHandler.__init__(self, connector) FSMAware.__init__(self) def _event_callback(self, message): self.process_message(message) def handle_fsm_step(self, message): - self.handle_event(message, self.server_connector) + self.handle_event(message, self.connector) diff --git a/tfw/internals/event_handling/test_event_handler.py b/tfw/internals/event_handling/test_event_handler.py index 049bb78..03f4141 100644 --- a/tfw/internals/event_handling/test_event_handler.py +++ b/tfw/internals/event_handling/test_event_handler.py @@ -9,11 +9,11 @@ from .event_handler import EventHandler class MockEventHandlerFactory(EventHandlerFactoryBase): - def _build_server_connector(self): - return MockServerConnector() + def _build_connector(self): + return MockConnector() -class MockServerConnector: +class MockConnector: def __init__(self): self.keys = [] self._on_message = None @@ -40,7 +40,7 @@ class MockServerConnector: class MockEventHandlerStub: def __init__(self): - self.server_connector = None + self.connector = None self.last_message = None self.cleaned_up = False self.started = False @@ -54,12 +54,12 @@ class MockEventHandlerStub: class MockEventHandler(MockEventHandlerStub): # pylint: disable=unused-argument - def handle_event(self, message, server_connector): + def handle_event(self, message, connector): self.last_message = message class MockCallable(MockEventHandlerStub): - def __call__(self, message, server_connector): + def __call__(self, message, connector): self.last_message = message @@ -78,17 +78,17 @@ def test_keys(): def test_build_from_object(test_keys, test_msg): mock_eh = MockEventHandlerStub() - def handle_event(message, server_connector): - raise RuntimeError(message, server_connector.keys) + def handle_event(message, connector): + raise RuntimeError(message, connector.keys) mock_eh.handle_event = handle_event assert not mock_eh.started eh = MockEventHandlerFactory().build(mock_eh, keys=test_keys) assert mock_eh.started - assert mock_eh.server_connector is eh.server_connector + assert mock_eh.connector is eh.connector with pytest.raises(RuntimeError) as err: - eh.server_connector.simulate_message(test_msg) + eh.connector.simulate_message(test_msg) msg, keys = err.args assert msg == test_msg assert keys == test_keys @@ -104,11 +104,11 @@ def test_build_from_object_with_keys(test_keys, test_msg): assert not mock_eh.started eh = MockEventHandlerFactory().build(mock_eh) - assert mock_eh.server_connector.keys == test_keys - assert eh.server_connector is mock_eh.server_connector + assert mock_eh.connector.keys == test_keys + assert eh.connector is mock_eh.connector assert mock_eh.started assert not mock_eh.last_message - eh.server_connector.simulate_message(test_msg) + eh.connector.simulate_message(test_msg) assert mock_eh.last_message == test_msg assert not mock_eh.cleaned_up EventHandler.stop_all_instances() @@ -118,14 +118,14 @@ def test_build_from_object_with_keys(test_keys, test_msg): def test_build_from_simple_object(test_keys, test_msg): class SimpleMockEventHandler: # pylint: disable=no-self-use - def handle_event(self, message, server_connector): - raise RuntimeError(message, server_connector) + def handle_event(self, message, connector): + raise RuntimeError(message, connector) mock_eh = SimpleMockEventHandler() eh = MockEventHandlerFactory().build(mock_eh, keys=test_keys) with pytest.raises(RuntimeError) as err: - eh.server_connector.simulate_message(test_msg) + eh.connector.simulate_message(test_msg) msg, keys = err.args assert msg == test_msg assert keys == test_keys @@ -138,10 +138,10 @@ def test_build_from_callable(test_keys, test_msg): eh = MockEventHandlerFactory().build(mock_eh, keys=test_keys) assert mock_eh.started - assert mock_eh.server_connector is eh.server_connector - assert eh.server_connector.keys == test_keys + assert mock_eh.connector is eh.connector + assert eh.connector.keys == test_keys assert not mock_eh.last_message - eh.server_connector.simulate_message(test_msg) + eh.connector.simulate_message(test_msg) assert mock_eh.last_message == test_msg assert not mock_eh.cleaned_up eh.stop() @@ -149,13 +149,13 @@ def test_build_from_callable(test_keys, test_msg): def test_build_from_function(test_keys, test_msg): - def some_function(message, server_connector): - raise RuntimeError(message, server_connector.keys) + def some_function(message, connector): + raise RuntimeError(message, connector.keys) eh = MockEventHandlerFactory().build(some_function, keys=test_keys) - assert eh.server_connector.keys == test_keys + assert eh.connector.keys == test_keys with pytest.raises(RuntimeError) as err: - eh.server_connector.simulate_message(test_msg) + eh.connector.simulate_message(test_msg) msg, keys = err.args assert msg == test_msg assert keys == test_keys @@ -166,7 +166,7 @@ def test_build_from_lambda(test_keys, test_msg): assert msg == test_msg fun = lambda msg, sc: assert_messages_equal(msg) eh = MockEventHandlerFactory().build(fun, keys=test_keys) - eh.server_connector.simulate_message(test_msg) + eh.connector.simulate_message(test_msg) def test_build_raises_if_no_key(test_keys): diff --git a/tfw/internals/networking/__init__.py b/tfw/internals/networking/__init__.py index 62766cd..dca7b24 100644 --- a/tfw/internals/networking/__init__.py +++ b/tfw/internals/networking/__init__.py @@ -1,4 +1,4 @@ from .serialization import serialize_tfw_msg, deserialize_tfw_msg, with_deserialize_tfw_msg, message_bytes -from .server_connector import ServerUplinkConnector, ServerDownlinkConnector, ServerConnector -from .event_handler_connector import EventHandlerConnector +from .zmq_connector import ZMQConnector, ZMQDownlinkConnector, ZMQUplinkConnector +from .zmq_listener import ZMQListener from .scope import Scope diff --git a/tfw/internals/networking/test_networking.py b/tfw/internals/networking/test_networking.py index cc23069..9bd35af 100644 --- a/tfw/internals/networking/test_networking.py +++ b/tfw/internals/networking/test_networking.py @@ -7,34 +7,34 @@ from tempfile import TemporaryDirectory import pytest from tornado.ioloop import IOLoop -from tfw.internals.networking import EventHandlerConnector, ServerConnector +from tfw.internals.networking import ZMQListener, ZMQConnector @pytest.fixture -def _connectors(): +def _listener_and_connector(): with TemporaryDirectory() as tmpdir: down_sock = join(tmpdir, 'down') up_sock = join(tmpdir, 'up') server_downlink = f'ipc://{down_sock}' server_uplink = f'ipc://{up_sock}' - ec = EventHandlerConnector(server_downlink, server_uplink) - sc = ServerConnector(server_uplink, server_downlink) - yield ec, sc - sc.close() - ec.close() + listener = ZMQListener(server_downlink, server_uplink) + connector = ZMQConnector(server_uplink, server_downlink) + yield listener, connector + listener.close() + connector.close() @pytest.fixture -def eh_connector(_connectors): - eh_connector, _ = _connectors - yield eh_connector +def zmq_listener(_listener_and_connector): + listener, _ = _listener_and_connector + yield listener @pytest.fixture -def server_connector(_connectors): - _, server_connector = _connectors - yield server_connector +def zmq_connector(_listener_and_connector): + _, connector = _listener_and_connector + yield connector def run_ioloop_once(): @@ -63,42 +63,42 @@ def test_messages(): ] -def test_server_downlink(eh_connector, server_connector, test_messages): +def test_server_downlink(zmq_listener, zmq_connector, test_messages): messages = [] - eh_connector.register_callback(messages.append) + zmq_listener.register_callback(messages.append) for message in test_messages: - server_connector.send_message(message) + zmq_connector.send_message(message) run_ioloop_once() assert messages == test_messages -def test_server_uplink(eh_connector, server_connector, test_messages): +def test_server_uplink(zmq_listener, zmq_connector, test_messages): messages = [] - server_connector.subscribe('') - server_connector.register_callback(messages.append) + zmq_connector.subscribe('') + zmq_connector.register_callback(messages.append) for message in test_messages: - eh_connector.send_message(message) + zmq_listener.send_message(message) run_ioloop_once() assert messages == test_messages -def test_connector_downlink_subscribe(eh_connector, server_connector): +def test_connector_downlink_subscribe(zmq_listener, zmq_connector): key1_messages = [{'key': '1', 'data': i} for i in range(randint(128, 256))] key2_messages = [{'key': '2', 'data': i} for i in range(randint(128, 256))] all_messages = key1_messages + key2_messages messages = [] - server_connector.subscribe('1') - server_connector.register_callback(messages.append) + zmq_connector.subscribe('1') + zmq_connector.register_callback(messages.append) for message in all_messages: - eh_connector.send_message(message) + zmq_listener.send_message(message) run_ioloop_once() diff --git a/tfw/internals/networking/server_connector.py b/tfw/internals/networking/zmq_connector.py similarity index 83% rename from tfw/internals/networking/server_connector.py rename to tfw/internals/networking/zmq_connector.py index 0ab1644..a701c99 100644 --- a/tfw/internals/networking/server_connector.py +++ b/tfw/internals/networking/zmq_connector.py @@ -9,7 +9,7 @@ from .serialization import serialize_tfw_msg, with_deserialize_tfw_msg LOG = logging.getLogger(__name__) -class ServerDownlinkConnector: +class ZMQDownlinkConnector: def __init__(self, connect_addr): self.keys = [] self._on_recv_callback = None @@ -41,7 +41,7 @@ class ServerDownlinkConnector: self._zmq_sub_stream.close() -class ServerUplinkConnector: +class ZMQUplinkConnector: def __init__(self, connect_addr): self._zmq_push_socket = zmq.Context.instance().socket(zmq.PUSH) self._zmq_push_socket.setsockopt(zmq.SNDHWM, 0) @@ -55,11 +55,11 @@ class ServerUplinkConnector: self._zmq_push_socket.close() -class ServerConnector(ServerDownlinkConnector, ServerUplinkConnector): +class ZMQConnector(ZMQDownlinkConnector, ZMQUplinkConnector): def __init__(self, downlink_connect_addr, uplink_connect_addr): - ServerDownlinkConnector.__init__(self, downlink_connect_addr) - ServerUplinkConnector.__init__(self, uplink_connect_addr) + ZMQDownlinkConnector.__init__(self, downlink_connect_addr) + ZMQUplinkConnector.__init__(self, uplink_connect_addr) def close(self): - ServerDownlinkConnector.close(self) - ServerUplinkConnector.close(self) + ZMQDownlinkConnector.close(self) + ZMQUplinkConnector.close(self) diff --git a/tfw/internals/networking/event_handler_connector.py b/tfw/internals/networking/zmq_listener.py similarity index 75% rename from tfw/internals/networking/event_handler_connector.py rename to tfw/internals/networking/zmq_listener.py index 74b6270..a46fb5a 100644 --- a/tfw/internals/networking/event_handler_connector.py +++ b/tfw/internals/networking/zmq_listener.py @@ -8,7 +8,7 @@ from .serialization import serialize_tfw_msg, with_deserialize_tfw_msg LOG = logging.getLogger(__name__) -class EventHandlerDownlinkConnector: +class ZMQDownlinkListener: def __init__(self, bind_addr): self._zmq_pull_socket = zmq.Context.instance().socket(zmq.PULL) self._zmq_pull_socket.setsockopt(zmq.RCVHWM, 0) @@ -24,7 +24,7 @@ class EventHandlerDownlinkConnector: self._zmq_pull_stream.close() -class EventHandlerUplinkConnector: +class ZMQUplinkListener: def __init__(self, bind_addr): self._zmq_pub_socket = zmq.Context.instance().socket(zmq.PUB) self._zmq_pub_socket.setsockopt(zmq.SNDHWM, 0) @@ -38,11 +38,11 @@ class EventHandlerUplinkConnector: self._zmq_pub_socket.close() -class EventHandlerConnector(EventHandlerDownlinkConnector, EventHandlerUplinkConnector): +class ZMQListener(ZMQDownlinkListener, ZMQUplinkListener): def __init__(self, downlink_bind_addr, uplink_bind_addr): - EventHandlerDownlinkConnector.__init__(self, downlink_bind_addr) - EventHandlerUplinkConnector.__init__(self, uplink_bind_addr) + ZMQDownlinkListener.__init__(self, downlink_bind_addr) + ZMQUplinkListener.__init__(self, uplink_bind_addr) def close(self): - EventHandlerDownlinkConnector.close(self) - EventHandlerUplinkConnector.close(self) + ZMQDownlinkListener.close(self) + ZMQUplinkListener.close(self) diff --git a/tfw/internals/server/zmq_websocket_router.py b/tfw/internals/server/zmq_websocket_router.py index 3f5e7fc..7d8b3f5 100644 --- a/tfw/internals/server/zmq_websocket_router.py +++ b/tfw/internals/server/zmq_websocket_router.py @@ -13,11 +13,11 @@ class ZMQWebSocketRouter(WebSocketHandler): instances = set() def initialize(self, **kwargs): - self.event_handler_connector = kwargs['event_handler_connector'] + self._listener = kwargs['listener'] self.tfw_router = TFWRouter(self.send_to_zmq, self.send_to_websockets) def send_to_zmq(self, message): - self.event_handler_connector.send_message(message) + self._listener.send_message(message) @classmethod def send_to_websockets(cls, message): @@ -32,7 +32,7 @@ class ZMQWebSocketRouter(WebSocketHandler): def open(self, *args, **kwargs): LOG.debug('WebSocket connection initiated!') - self.event_handler_connector.register_callback(self.zmq_callback) + self._listener.register_callback(self.zmq_callback) def zmq_callback(self, message): LOG.debug('Received on ZMQ pull socket: %s', message) diff --git a/tfw/main/event_handler_factory.py b/tfw/main/event_handler_factory.py index c2c4045..b913898 100644 --- a/tfw/main/event_handler_factory.py +++ b/tfw/main/event_handler_factory.py @@ -4,5 +4,5 @@ from .tfw_connector import TFWConnector class EventHandlerFactory(EventHandlerFactoryBase): - def _build_server_connector(self): + def _build_connector(self): return TFWConnector() diff --git a/tfw/main/tfw_connector.py b/tfw/main/tfw_connector.py index 5324ee6..be21dab 100644 --- a/tfw/main/tfw_connector.py +++ b/tfw/main/tfw_connector.py @@ -1,4 +1,4 @@ -from tfw.internals.networking import ServerConnector, ServerUplinkConnector +from tfw.internals.networking import ZMQConnector, ZMQUplinkConnector from tfw.config import TFWENV @@ -12,12 +12,12 @@ class ConnAddrMixin: return f'tcp://localhost:{TFWENV.PUB_PORT}' -class TFWUplinkConnector(ServerUplinkConnector, ConnAddrMixin): +class TFWUplinkConnector(ZMQUplinkConnector, ConnAddrMixin): def __init__(self): super().__init__(self.uplink_conn_addr) -class TFWConnector(ServerConnector, ConnAddrMixin): +class TFWConnector(ZMQConnector, ConnAddrMixin): def __init__(self): super().__init__( self.downlink_conn_addr, diff --git a/tfw/main/tfw_server.py b/tfw/main/tfw_server.py index 4cc1c5e..1706417 100644 --- a/tfw/main/tfw_server.py +++ b/tfw/main/tfw_server.py @@ -2,7 +2,7 @@ import logging from tornado.web import Application -from tfw.internals.networking import EventHandlerConnector +from tfw.internals.networking import ZMQListener from tfw.internals.server import ZMQWebSocketRouter from tfw.config import TFWENV @@ -16,13 +16,13 @@ class TFWServer: SUB socket. """ def __init__(self): - self._event_handler_connector = EventHandlerConnector( + self._listener = ZMQListener( downlink_bind_addr=f'tcp://*:{TFWENV.PULL_PORT}', uplink_bind_addr=f'tcp://*:{TFWENV.PUB_PORT}' ) self.application = Application([( r'/ws', ZMQWebSocketRouter, { - 'event_handler_connector': self._event_handler_connector, + 'listener': self._listener, } )]) From 78c3a8cf98f74b0b15a298d1b6dc8e7c2b7a8e21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 30 Jul 2019 15:23:24 +0200 Subject: [PATCH 096/174] Fix socket bind order regarding ZMQStream --- tfw/internals/networking/zmq_listener.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfw/internals/networking/zmq_listener.py b/tfw/internals/networking/zmq_listener.py index a46fb5a..a1e1313 100644 --- a/tfw/internals/networking/zmq_listener.py +++ b/tfw/internals/networking/zmq_listener.py @@ -12,8 +12,8 @@ class ZMQDownlinkListener: def __init__(self, bind_addr): self._zmq_pull_socket = zmq.Context.instance().socket(zmq.PULL) self._zmq_pull_socket.setsockopt(zmq.RCVHWM, 0) - self._zmq_pull_stream = ZMQStream(self._zmq_pull_socket) self._zmq_pull_socket.bind(bind_addr) + self._zmq_pull_stream = ZMQStream(self._zmq_pull_socket) LOG.debug('Pull socket bound to %s', bind_addr) def register_callback(self, callback): From 911831fdb1f719b193ef0491aa195e1c554f972c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 31 Jul 2019 17:08:47 +0200 Subject: [PATCH 097/174] Implement sync API for networking --- tfw/internals/networking/test_networking.py | 46 +++++++++++++++++++++ tfw/internals/networking/zmq_connector.py | 22 ++++++++-- tfw/internals/networking/zmq_listener.py | 17 +++++++- 3 files changed, 80 insertions(+), 5 deletions(-) diff --git a/tfw/internals/networking/test_networking.py b/tfw/internals/networking/test_networking.py index 9bd35af..d1d4261 100644 --- a/tfw/internals/networking/test_networking.py +++ b/tfw/internals/networking/test_networking.py @@ -3,6 +3,7 @@ from os.path import join from secrets import token_urlsafe from random import randint from tempfile import TemporaryDirectory +from contextlib import suppress import pytest from tornado.ioloop import IOLoop @@ -63,6 +64,26 @@ def test_messages(): ] +def wait_until_subscriber_connects(listener, connector): + # Warning: you are better off without comprehending how this works + # Reference: ZMQ PUB-SUB slow joiner problem + + # Wait until something can go through the connection + dummy = {'key': '-'} + while True: + listener.send_message(dummy) + with suppress(IOError): + if connector.recv_message(block=False) == dummy: + break + # Throw away leftover messages from last while loop + sentinel = {'key': '_'} + listener.send_message(sentinel) + while True: + with suppress(IOError): + if connector.recv_message(block=False) == sentinel: + break + + def test_server_downlink(zmq_listener, zmq_connector, test_messages): messages = [] zmq_listener.register_callback(messages.append) @@ -104,3 +125,28 @@ def test_connector_downlink_subscribe(zmq_listener, zmq_connector): assert messages == key1_messages assert all((msg not in messages for msg in key2_messages)) + + +def test_listener_sync_recv(zmq_listener, zmq_connector, test_messages): + for message in test_messages: + zmq_connector.send_message(message) + assert zmq_listener.recv_message() == message + + +def test_connector_sync_recv(zmq_listener, zmq_connector, test_messages): + zmq_connector.subscribe('') + wait_until_subscriber_connects(zmq_listener, zmq_connector) + for message in test_messages: + zmq_listener.send_message(message) + assert zmq_connector.recv_message() == message + + +def test_sync_recv_raises_if_callback_is_registered(zmq_listener, zmq_connector): + zmq_listener.register_callback(lambda msg: None) + zmq_connector.register_callback(lambda msg: None) + + with pytest.raises(RuntimeError): + zmq_listener.recv_message() + + with pytest.raises(RuntimeError): + zmq_connector.recv_message() diff --git a/tfw/internals/networking/zmq_connector.py b/tfw/internals/networking/zmq_connector.py index a701c99..7dfa77c 100644 --- a/tfw/internals/networking/zmq_connector.py +++ b/tfw/internals/networking/zmq_connector.py @@ -4,7 +4,11 @@ import zmq from zmq.eventloop.zmqstream import ZMQStream from .scope import Scope -from .serialization import serialize_tfw_msg, with_deserialize_tfw_msg +from .serialization import ( + serialize_tfw_msg, + deserialize_tfw_msg, + with_deserialize_tfw_msg +) LOG = logging.getLogger(__name__) @@ -29,8 +33,20 @@ class ZMQDownlinkConnector: self.keys.remove(key) def register_callback(self, callback): - self._on_recv_callback = callback - self._zmq_sub_stream.on_recv(with_deserialize_tfw_msg(self._on_recv)) + if callback: + self._on_recv_callback = callback + self._zmq_sub_stream.on_recv(with_deserialize_tfw_msg(self._on_recv)) + else: + self._zmq_sub_stream.on_recv(None) + + def recv_message(self, *, block=True): + if self._zmq_sub_stream.receiving(): + raise RuntimeError('Synchronous recv() called while a callback is registered!') + flags = 0 if block else zmq.NOBLOCK + try: + return deserialize_tfw_msg(*self._zmq_sub_socket.recv_multipart(flags)) + except zmq.ZMQError: + raise IOError("No data available to recv!") def _on_recv(self, message): key = message['key'] diff --git a/tfw/internals/networking/zmq_listener.py b/tfw/internals/networking/zmq_listener.py index a1e1313..d43814a 100644 --- a/tfw/internals/networking/zmq_listener.py +++ b/tfw/internals/networking/zmq_listener.py @@ -3,7 +3,11 @@ import logging import zmq from zmq.eventloop.zmqstream import ZMQStream -from .serialization import serialize_tfw_msg, with_deserialize_tfw_msg +from .serialization import ( + serialize_tfw_msg, + deserialize_tfw_msg, + with_deserialize_tfw_msg +) LOG = logging.getLogger(__name__) @@ -17,9 +21,18 @@ class ZMQDownlinkListener: LOG.debug('Pull socket bound to %s', bind_addr) def register_callback(self, callback): - callback = with_deserialize_tfw_msg(callback) + callback = with_deserialize_tfw_msg(callback) if callback else None self._zmq_pull_stream.on_recv(callback) + def recv_message(self, *, block=True): + if self._zmq_pull_stream.receiving(): + raise RuntimeError('Synchronous recv() called while a callback is registered!') + flags = 0 if block else zmq.NOBLOCK + try: + return deserialize_tfw_msg(*self._zmq_pull_socket.recv_multipart(flags)) + except zmq.ZMQError: + raise IOError("No data available to recv!") + def close(self): self._zmq_pull_stream.close() From 25bd9aa0f34b4035c73a9b396938aeb94176c473 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 31 Jul 2019 16:30:06 +0200 Subject: [PATCH 098/174] Implement reference counting mechanism --- tfw/internals/crypto/__init__.py | 2 + .../{crypto.py => crypto/authentication.py} | 36 ---------- tfw/internals/crypto/key_manager.py | 48 +++++++++++++ tfw/internals/ref_counter/__init__.py | 1 + tfw/internals/ref_counter/ref_counter.py | 40 +++++++++++ tfw/internals/ref_counter/test_ref_counter.py | 69 +++++++++++++++++++ 6 files changed, 160 insertions(+), 36 deletions(-) create mode 100644 tfw/internals/crypto/__init__.py rename tfw/internals/{crypto.py => crypto/authentication.py} (69%) create mode 100644 tfw/internals/crypto/key_manager.py create mode 100644 tfw/internals/ref_counter/__init__.py create mode 100644 tfw/internals/ref_counter/ref_counter.py create mode 100644 tfw/internals/ref_counter/test_ref_counter.py diff --git a/tfw/internals/crypto/__init__.py b/tfw/internals/crypto/__init__.py new file mode 100644 index 0000000..0ff122a --- /dev/null +++ b/tfw/internals/crypto/__init__.py @@ -0,0 +1,2 @@ +from .authentication import sign_message, verify_message +from .key_manager import KeyManager diff --git a/tfw/internals/crypto.py b/tfw/internals/crypto/authentication.py similarity index 69% rename from tfw/internals/crypto.py rename to tfw/internals/crypto/authentication.py index 04aff16..26e9e11 100644 --- a/tfw/internals/crypto.py +++ b/tfw/internals/crypto/authentication.py @@ -1,10 +1,6 @@ from functools import wraps from base64 import b64encode, b64decode from copy import deepcopy -from hashlib import md5 -from os import urandom, chmod -from os.path import exists -from stat import S_IRUSR, S_IWUSR, S_IXUSR from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.hashes import SHA256 @@ -12,12 +8,6 @@ from cryptography.hazmat.primitives.hmac import HMAC as _HMAC from cryptography.exceptions import InvalidSignature from tfw.internals.networking import message_bytes -from tfw.internals.lazy import lazy_property -from tfw.config import TFWENV - - -def message_checksum(message): - return md5(message_bytes(message)).hexdigest() def sign_message(key, message): @@ -43,32 +33,6 @@ def verify_message(key, message): return False -class KeyManager: - def __init__(self): - self.keyfile = TFWENV.AUTH_KEY - if not exists(self.keyfile): - self._init_auth_key() - - @lazy_property - def auth_key(self): - with open(self.keyfile, 'rb') as ifile: - return ifile.read() - - def _init_auth_key(self): - key = self.generate_key() - with open(self.keyfile, 'wb') as ofile: - ofile.write(key) - self._chmod_700_keyfile() - return key - - @staticmethod - def generate_key(): - return urandom(32) - - def _chmod_700_keyfile(self): - chmod(self.keyfile, S_IRUSR | S_IWUSR | S_IXUSR) - - class HMAC: def __init__(self, key, message): self.key = key diff --git a/tfw/internals/crypto/key_manager.py b/tfw/internals/crypto/key_manager.py new file mode 100644 index 0000000..d177f11 --- /dev/null +++ b/tfw/internals/crypto/key_manager.py @@ -0,0 +1,48 @@ +from atexit import register +from tempfile import gettempdir +from os import urandom, chmod, remove +from os.path import exists, join +from stat import S_IRUSR, S_IWUSR, S_IXUSR + +from tfw.internals.lazy import lazy_property +from tfw.internals.ref_counter import RefCounter + + +KEYFILE = join(gettempdir(), 'tfw-auth.key') +LOCKFILE = join(gettempdir(), 'tfw-auth.lock') + + +class KeyManagerRefCounter(RefCounter): + def deallocate(self): + if exists(KEYFILE): + remove(KEYFILE) + + +class KeyManager: + keyfile = KEYFILE + refcounter = KeyManagerRefCounter(LOCKFILE) + + def __init__(self): + if not exists(self.keyfile): + self._init_auth_key() + + @lazy_property + def auth_key(self): + with open(self.keyfile, 'rb') as ifile: + return ifile.read() + + def _init_auth_key(self): + key = self.generate_key() + with open(self.keyfile, 'wb') as ofile: + ofile.write(key) + self._chmod_700_keyfile() + return key + + @staticmethod + def generate_key(): + return urandom(32) + + def _chmod_700_keyfile(self): + chmod(self.keyfile, S_IRUSR | S_IWUSR | S_IXUSR) + +register(KeyManager.refcounter.teardown_instance) diff --git a/tfw/internals/ref_counter/__init__.py b/tfw/internals/ref_counter/__init__.py new file mode 100644 index 0000000..25f2d50 --- /dev/null +++ b/tfw/internals/ref_counter/__init__.py @@ -0,0 +1 @@ +from .ref_counter import RefCounter diff --git a/tfw/internals/ref_counter/ref_counter.py b/tfw/internals/ref_counter/ref_counter.py new file mode 100644 index 0000000..093fd53 --- /dev/null +++ b/tfw/internals/ref_counter/ref_counter.py @@ -0,0 +1,40 @@ +from os import remove +from fcntl import flock, LOCK_EX, LOCK_UN + + +class RefCounter: + def __init__(self, lockpath): + self.lockpath = lockpath + self._lockfile = open(self.lockpath, 'a+') + flock(self._lockfile, LOCK_EX) + counter = self._read_counter() + self._write_counter(counter+1) + flock(self._lockfile, LOCK_UN) + + def _read_counter(self): + self._lockfile.seek(0) + try: + counter = int(self._lockfile.read()) + except ValueError: + counter = 0 + return counter + + def _write_counter(self, counter): + self._lockfile.seek(0) + self._lockfile.truncate() + self._lockfile.write(str(counter)) + self._lockfile.flush() + + def teardown_instance(self): + flock(self._lockfile, LOCK_EX) + counter = self._read_counter() + if counter <= 1: + remove(self.lockpath) + self.deallocate() + else: + self._write_counter(counter-1) + flock(self._lockfile, LOCK_UN) + self._lockfile.close() + + def deallocate(self): + pass diff --git a/tfw/internals/ref_counter/test_ref_counter.py b/tfw/internals/ref_counter/test_ref_counter.py new file mode 100644 index 0000000..0666ad8 --- /dev/null +++ b/tfw/internals/ref_counter/test_ref_counter.py @@ -0,0 +1,69 @@ +# pylint: disable=redefined-outer-name +from dataclasses import dataclass +from textwrap import dedent +from os import mkfifo +from os.path import join +from signal import SIGINT +from subprocess import DEVNULL, Popen, PIPE +from tempfile import TemporaryDirectory + +import pytest + +from .ref_counter import RefCounter + + +@dataclass +class CounterContext: + lockpath: str + pipepath: str + + @property + def source(self): + return dedent(f'''\ + from time import sleep + from atexit import register + from ref_counter import RefCounter + + counter = RefCounter('{self.lockpath}') + register(counter.teardown_instance) + print(flush=True) + while True: + sleep(1) + ''') + + +@pytest.fixture +def context(): + with TemporaryDirectory() as workdir: + pipepath = join(workdir, 'test.pipe') + mkfifo(pipepath) + yield CounterContext(join(workdir, 'test.lock'), pipepath) + +def test_increment_decrement(context): + counter, processes = 0, [] + for _ in range(5): + new_proc = Popen(['python3', '-c', context.source], stdout=PIPE, stderr=DEVNULL) + new_proc.stdout.readline() + processes.append(new_proc) + counter += 1 + for proc in processes: + with open(context.lockpath, 'r') as lock: + assert lock.read() == str(counter) + counter -= 1 + proc.send_signal(SIGINT) + proc.wait() + +def test_deallocate(context): + state = False + def trigger(): + nonlocal state + state = True + refcounters = [] + for _ in range(32): + new_refc = RefCounter(context.lockpath) + new_refc.deallocate = trigger + refcounters.append(new_refc) + for refc in refcounters: + assert not state + refc.teardown_instance() + assert state From 96c6c9b3585bdff321306e367b13b3152b04b3a9 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 31 Jul 2019 16:59:08 +0200 Subject: [PATCH 099/174] Correct whitespaces and add context manager for file locking --- tfw/internals/crypto/key_manager.py | 2 +- tfw/internals/ref_counter/ref_counter.py | 25 +++++++++++-------- tfw/internals/ref_counter/test_ref_counter.py | 2 ++ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/tfw/internals/crypto/key_manager.py b/tfw/internals/crypto/key_manager.py index d177f11..4e56cad 100644 --- a/tfw/internals/crypto/key_manager.py +++ b/tfw/internals/crypto/key_manager.py @@ -7,7 +7,6 @@ from stat import S_IRUSR, S_IWUSR, S_IXUSR from tfw.internals.lazy import lazy_property from tfw.internals.ref_counter import RefCounter - KEYFILE = join(gettempdir(), 'tfw-auth.key') LOCKFILE = join(gettempdir(), 'tfw-auth.lock') @@ -45,4 +44,5 @@ class KeyManager: def _chmod_700_keyfile(self): chmod(self.keyfile, S_IRUSR | S_IWUSR | S_IXUSR) + register(KeyManager.refcounter.teardown_instance) diff --git a/tfw/internals/ref_counter/ref_counter.py b/tfw/internals/ref_counter/ref_counter.py index 093fd53..4ff7889 100644 --- a/tfw/internals/ref_counter/ref_counter.py +++ b/tfw/internals/ref_counter/ref_counter.py @@ -1,4 +1,5 @@ from os import remove +from contextlib import contextmanager from fcntl import flock, LOCK_EX, LOCK_UN @@ -6,9 +7,14 @@ class RefCounter: def __init__(self, lockpath): self.lockpath = lockpath self._lockfile = open(self.lockpath, 'a+') + with self.locked(): + counter = self._read_counter() + self._write_counter(counter+1) + + @contextmanager + def locked(self): flock(self._lockfile, LOCK_EX) - counter = self._read_counter() - self._write_counter(counter+1) + yield flock(self._lockfile, LOCK_UN) def _read_counter(self): @@ -26,14 +32,13 @@ class RefCounter: self._lockfile.flush() def teardown_instance(self): - flock(self._lockfile, LOCK_EX) - counter = self._read_counter() - if counter <= 1: - remove(self.lockpath) - self.deallocate() - else: - self._write_counter(counter-1) - flock(self._lockfile, LOCK_UN) + with self.locked(): + counter = self._read_counter() + if counter <= 1: + remove(self.lockpath) + self.deallocate() + else: + self._write_counter(counter-1) self._lockfile.close() def deallocate(self): diff --git a/tfw/internals/ref_counter/test_ref_counter.py b/tfw/internals/ref_counter/test_ref_counter.py index 0666ad8..3b563b9 100644 --- a/tfw/internals/ref_counter/test_ref_counter.py +++ b/tfw/internals/ref_counter/test_ref_counter.py @@ -39,6 +39,7 @@ def context(): mkfifo(pipepath) yield CounterContext(join(workdir, 'test.lock'), pipepath) + def test_increment_decrement(context): counter, processes = 0, [] for _ in range(5): @@ -53,6 +54,7 @@ def test_increment_decrement(context): proc.send_signal(SIGINT) proc.wait() + def test_deallocate(context): state = False def trigger(): From 38eec06b4d068e1fb51a8b8b7b1e21673c335a30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 31 Jul 2019 17:36:03 +0200 Subject: [PATCH 100/174] Move ZMQ bind address logging to TFWServer --- tfw/internals/networking/zmq_listener.py | 2 -- tfw/main/tfw_server.py | 8 ++++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tfw/internals/networking/zmq_listener.py b/tfw/internals/networking/zmq_listener.py index d43814a..2cf6b3d 100644 --- a/tfw/internals/networking/zmq_listener.py +++ b/tfw/internals/networking/zmq_listener.py @@ -18,7 +18,6 @@ class ZMQDownlinkListener: self._zmq_pull_socket.setsockopt(zmq.RCVHWM, 0) self._zmq_pull_socket.bind(bind_addr) self._zmq_pull_stream = ZMQStream(self._zmq_pull_socket) - LOG.debug('Pull socket bound to %s', bind_addr) def register_callback(self, callback): callback = with_deserialize_tfw_msg(callback) if callback else None @@ -42,7 +41,6 @@ class ZMQUplinkListener: self._zmq_pub_socket = zmq.Context.instance().socket(zmq.PUB) self._zmq_pub_socket.setsockopt(zmq.SNDHWM, 0) self._zmq_pub_socket.bind(bind_addr) - LOG.debug('Pub socket bound to %s', bind_addr) def send_message(self, message: dict): self._zmq_pub_socket.send_multipart(serialize_tfw_msg(message)) diff --git a/tfw/main/tfw_server.py b/tfw/main/tfw_server.py index 1706417..8c64a8e 100644 --- a/tfw/main/tfw_server.py +++ b/tfw/main/tfw_server.py @@ -16,10 +16,14 @@ class TFWServer: SUB socket. """ def __init__(self): + downlink_bind_addr = f'tcp://*:{TFWENV.PULL_PORT}' + uplink_bind_addr = f'tcp://*:{TFWENV.PUB_PORT}' self._listener = ZMQListener( - downlink_bind_addr=f'tcp://*:{TFWENV.PULL_PORT}', - uplink_bind_addr=f'tcp://*:{TFWENV.PUB_PORT}' + downlink_bind_addr=downlink_bind_addr, + uplink_bind_addr=uplink_bind_addr ) + LOG.debug('Pull socket bound to %s', downlink_bind_addr) + LOG.debug('Pub socket bound to %s', uplink_bind_addr) self.application = Application([( r'/ws', ZMQWebSocketRouter, { 'listener': self._listener, From 0a218c8d6d2c9835494e182a2f0d4f047279d324 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 1 Aug 2019 15:12:58 +0200 Subject: [PATCH 101/174] Fix relative path problem --- tfw/internals/ref_counter/test_ref_counter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tfw/internals/ref_counter/test_ref_counter.py b/tfw/internals/ref_counter/test_ref_counter.py index 3b563b9..6d65997 100644 --- a/tfw/internals/ref_counter/test_ref_counter.py +++ b/tfw/internals/ref_counter/test_ref_counter.py @@ -1,8 +1,8 @@ # pylint: disable=redefined-outer-name from dataclasses import dataclass from textwrap import dedent -from os import mkfifo -from os.path import join +from os import chdir, mkfifo +from os.path import dirname, join, realpath from signal import SIGINT from subprocess import DEVNULL, Popen, PIPE from tempfile import TemporaryDirectory @@ -35,6 +35,7 @@ class CounterContext: @pytest.fixture def context(): with TemporaryDirectory() as workdir: + chdir(dirname(realpath(__file__))) pipepath = join(workdir, 'test.pipe') mkfifo(pipepath) yield CounterContext(join(workdir, 'test.lock'), pipepath) From e87a16991c9111667715c80a6d2cd12c0607a9e4 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 5 Aug 2019 13:47:05 +0200 Subject: [PATCH 102/174] Implement signed event handler --- tfw/components/fsm/fsm_handler.py | 7 +------ tfw/event_handlers.py | 2 +- tfw/internals/event_handling/__init__.py | 3 ++- .../event_handling/signed_event_handler.py | 20 +++++++++++++++++++ 4 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 tfw/internals/event_handling/signed_event_handler.py diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index 18302d2..60b8c16 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -24,11 +24,10 @@ class FSMHandler: An 'fsm_update' message is broadcasted after every successful command. """ - def __init__(self, *, fsm_type, require_signature=False): + def __init__(self, *, fsm_type): self.fsm = fsm_type() self._fsm_updater = FSMUpdater(self.fsm) self.auth_key = KeyManager().auth_key - self._require_signature = require_signature self.command_handlers = { 'trigger': self.handle_trigger, @@ -54,10 +53,6 @@ class FSMHandler: the action to try triggering in data['value'] """ trigger = message['data']['value'] - if self._require_signature: - if not verify_message(self.auth_key, message): - LOG.error('Ignoring unsigned trigger command: %s', message) - return None if self.fsm.step(trigger): return message return None diff --git a/tfw/event_handlers.py b/tfw/event_handlers.py index 8dc872e..0ea1050 100644 --- a/tfw/event_handlers.py +++ b/tfw/event_handlers.py @@ -1,2 +1,2 @@ # pylint: disable=unused-import -from tfw.internals.event_handling import EventHandler, FSMAwareEventHandler +from tfw.internals.event_handling import EventHandler, FSMAwareEventHandler, SignedEventHandler diff --git a/tfw/internals/event_handling/__init__.py b/tfw/internals/event_handling/__init__.py index 2a33065..8238d21 100644 --- a/tfw/internals/event_handling/__init__.py +++ b/tfw/internals/event_handling/__init__.py @@ -1,3 +1,4 @@ -from .event_handler_factory_base import EventHandlerFactoryBase from .event_handler import EventHandler +from .event_handler_factory_base import EventHandlerFactoryBase from .fsm_aware_event_handler import FSMAwareEventHandler +from .signed_event_handler import SignedEventHandler diff --git a/tfw/internals/event_handling/signed_event_handler.py b/tfw/internals/event_handling/signed_event_handler.py new file mode 100644 index 0000000..1ee72b8 --- /dev/null +++ b/tfw/internals/event_handling/signed_event_handler.py @@ -0,0 +1,20 @@ +import logging + +from tfw.internals.crypto import KeyManager, verify_message + +from .event_handler import EventHandler + +LOG = logging.getLogger(__name__) + + +# pylint: disable=abstract-method +class SignedEventHandler(EventHandler): + def __init__(self, connector): + self._auth_key = KeyManager().auth_key + super().__init__(connector) + + def _event_callback(self, message): + if verify_message(self._auth_key, message): + self.handle_event(message, self.connector) + else: + LOG.error('Message does not have valid signature: %s', message) From e6d277752094292a41de73cda03a078431ecd019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 6 Aug 2019 11:33:26 +0200 Subject: [PATCH 103/174] Replace pipe-io-server source with the new pip package --- requirements.txt | 1 + tfw/components/pipe_io/pipe_io_handler.py | 2 +- .../pipe_io/pipe_io_server/__init__.py | 4 - .../pipe_io/pipe_io_server/deque.py | 27 --- tfw/components/pipe_io/pipe_io_server/pipe.py | 18 -- .../pipe_io/pipe_io_server/pipe_io_server.py | 25 --- .../pipe_io/pipe_io_server/pipe_io_thread.py | 45 ----- .../pipe_io_server/pipe_reader_server.py | 39 ---- .../pipe_io_server/pipe_reader_thread.py | 44 ----- .../pipe_io_server/pipe_writer_server.py | 38 ---- .../pipe_io_server/pipe_writer_thread.py | 50 ----- .../terminate_process_on_failure.py | 15 -- .../pipe_io_server/test_pipe_io_server.py | 187 ------------------ 13 files changed, 2 insertions(+), 493 deletions(-) delete mode 100644 tfw/components/pipe_io/pipe_io_server/__init__.py delete mode 100644 tfw/components/pipe_io/pipe_io_server/deque.py delete mode 100644 tfw/components/pipe_io/pipe_io_server/pipe.py delete mode 100644 tfw/components/pipe_io/pipe_io_server/pipe_io_server.py delete mode 100644 tfw/components/pipe_io/pipe_io_server/pipe_io_thread.py delete mode 100644 tfw/components/pipe_io/pipe_io_server/pipe_reader_server.py delete mode 100644 tfw/components/pipe_io/pipe_io_server/pipe_reader_thread.py delete mode 100644 tfw/components/pipe_io/pipe_io_server/pipe_writer_server.py delete mode 100644 tfw/components/pipe_io/pipe_io_server/pipe_writer_thread.py delete mode 100644 tfw/components/pipe_io/pipe_io_server/terminate_process_on_failure.py delete mode 100644 tfw/components/pipe_io/pipe_io_server/test_pipe_io_server.py diff --git a/requirements.txt b/requirements.txt index 3c34b45..be5fcd3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ python-dateutil>=2.0.0,<3.0.0 pytest>=5.0.0,<6.0.0 pylint>=2.0.0,<3.0.0 rope>=0.0.0,<1.0.0 +pipe-io-server>=1.0.0,<2.0.0 diff --git a/tfw/components/pipe_io/pipe_io_handler.py b/tfw/components/pipe_io/pipe_io_handler.py index 74aabe8..671f1db 100644 --- a/tfw/components/pipe_io/pipe_io_handler.py +++ b/tfw/components/pipe_io/pipe_io_handler.py @@ -10,7 +10,7 @@ from secrets import token_urlsafe from threading import Thread from contextlib import suppress -from .pipe_io_server import PipeIOServer, terminate_process_on_failure +from pipe_io_server import PipeIOServer, terminate_process_on_failure LOG = logging.getLogger(__name__) DEFAULT_PERMISSIONS = 0o600 diff --git a/tfw/components/pipe_io/pipe_io_server/__init__.py b/tfw/components/pipe_io/pipe_io_server/__init__.py deleted file mode 100644 index b0d450a..0000000 --- a/tfw/components/pipe_io/pipe_io_server/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .pipe_io_server import PipeIOServer -from .pipe_reader_server import PipeReaderServer -from .pipe_writer_server import PipeWriterServer -from .terminate_process_on_failure import terminate_process_on_failure diff --git a/tfw/components/pipe_io/pipe_io_server/deque.py b/tfw/components/pipe_io/pipe_io_server/deque.py deleted file mode 100644 index b2f1ab4..0000000 --- a/tfw/components/pipe_io/pipe_io_server/deque.py +++ /dev/null @@ -1,27 +0,0 @@ -from collections import deque -from threading import Lock, Condition - - -class Deque: - def __init__(self): - self._queue = deque() - - self._mutex = Lock() - self._not_empty = Condition(self._mutex) - - def pop(self): - with self._mutex: - while not self._queue: - self._not_empty.wait() - return self._queue.pop() - - def push(self, item): - self._push(item, self._queue.appendleft) - - def push_front(self, item): - self._push(item, self._queue.append) - - def _push(self, item, put_method): - with self._mutex: - put_method(item) - self._not_empty.notify() diff --git a/tfw/components/pipe_io/pipe_io_server/pipe.py b/tfw/components/pipe_io/pipe_io_server/pipe.py deleted file mode 100644 index f83664a..0000000 --- a/tfw/components/pipe_io/pipe_io_server/pipe.py +++ /dev/null @@ -1,18 +0,0 @@ -from os import mkfifo, remove, chmod -from os.path import exists - -DEFAULT_PERMISSIONS = 0o600 - - -class Pipe: - def __init__(self, path): - self.path = path - - def recreate(self, permissions): - self.remove() - mkfifo(self.path) - chmod(self.path, permissions) # use chmod to ignore umask - - def remove(self): - if exists(self.path): - remove(self.path) diff --git a/tfw/components/pipe_io/pipe_io_server/pipe_io_server.py b/tfw/components/pipe_io/pipe_io_server/pipe_io_server.py deleted file mode 100644 index 8204adb..0000000 --- a/tfw/components/pipe_io/pipe_io_server/pipe_io_server.py +++ /dev/null @@ -1,25 +0,0 @@ -from .pipe import DEFAULT_PERMISSIONS -from .pipe_reader_server import PipeReaderServer -from .pipe_writer_server import PipeWriterServer - - -class PipeIOServer(PipeReaderServer, PipeWriterServer): - # pylint: disable=abstract-method - def __init__( - self, - in_pipe, - out_pipe, - permissions=DEFAULT_PERMISSIONS, - manage_pipes=True - ): - super().__init__( - in_pipe=in_pipe, - out_pipe=out_pipe, - permissions=permissions, - manage_pipes=manage_pipes - ) - - def _io_threads(self): - # pylint: disable=no-member - yield from PipeReaderServer._io_threads(self) - yield from PipeWriterServer._io_threads(self) diff --git a/tfw/components/pipe_io/pipe_io_server/pipe_io_thread.py b/tfw/components/pipe_io/pipe_io_server/pipe_io_thread.py deleted file mode 100644 index 91a50b4..0000000 --- a/tfw/components/pipe_io/pipe_io_server/pipe_io_thread.py +++ /dev/null @@ -1,45 +0,0 @@ -from threading import Thread, Event - -from .terminate_process_on_failure import terminate_process_on_failure - - -class PipeIOThread(Thread): - def __init__(self): - super().__init__(daemon=True) - self._stop_event = Event() - self.__io_threads = [] - - @terminate_process_on_failure - def run(self): - self.__io_threads.extend(self._io_threads()) - for thread in self.__io_threads: - thread.start() - self._stop_event.wait() - self._stop_threads() - - def _io_threads(self): - raise NotImplementedError() - - def _stop_threads(self): - for thread in self.__io_threads: - if thread.is_alive(): - thread.stop() - self.on_stop() - - def on_stop(self): - pass - - def stop(self): - self._stop_event.set() - if self.is_alive(): - self.join() - - def wait(self): - self._stop_event.wait() - - def __enter__(self): - self.start() - return self - - def __exit__(self, type_, value, tb): - self.stop() diff --git a/tfw/components/pipe_io/pipe_io_server/pipe_reader_server.py b/tfw/components/pipe_io/pipe_io_server/pipe_reader_server.py deleted file mode 100644 index e26bb87..0000000 --- a/tfw/components/pipe_io/pipe_io_server/pipe_reader_server.py +++ /dev/null @@ -1,39 +0,0 @@ -from .pipe import Pipe, DEFAULT_PERMISSIONS -from .pipe_io_thread import PipeIOThread -from .pipe_reader_thread import PipeReaderThread - - -class PipeReaderServer(PipeIOThread): - def __init__( - self, - in_pipe, - permissions=DEFAULT_PERMISSIONS, - manage_pipes=True, - **kwargs - ): - super().__init__(**kwargs) - self._reader_thread = None - self._manage_pipes = manage_pipes - self._in_pipe = in_pipe - if self._manage_pipes: - Pipe(self.in_pipe).recreate(permissions) - - @property - def in_pipe(self): - return self._in_pipe - - def _io_threads(self): - self._reader_thread = PipeReaderThread( - self.in_pipe, - self._stop_event, - self.handle_message - ) - yield self._reader_thread - - def handle_message(self, message): - raise NotImplementedError() - - def stop(self): - super().stop() - if self._manage_pipes: - Pipe(self.in_pipe).remove() diff --git a/tfw/components/pipe_io/pipe_io_server/pipe_reader_thread.py b/tfw/components/pipe_io/pipe_io_server/pipe_reader_thread.py deleted file mode 100644 index 4bce19d..0000000 --- a/tfw/components/pipe_io/pipe_io_server/pipe_reader_thread.py +++ /dev/null @@ -1,44 +0,0 @@ -from contextlib import suppress -from os import open as osopen -from os import write, close, O_WRONLY, O_NONBLOCK -from threading import Thread - -from .terminate_process_on_failure import terminate_process_on_failure - - -class PipeReaderThread(Thread): - eof = b'' - stop_sequence = b'stop_reading\n' - - def __init__(self, pipe_path, stop_event, message_handler): - super().__init__(daemon=True) - self._message_handler = message_handler - self._pipe_path = pipe_path - self._stop_event = stop_event - - @terminate_process_on_failure - def run(self): - with self._open() as pipe: - while True: - message = pipe.readline() - if message == self.stop_sequence: - self._stop_event.set() - break - if message == self.eof: - self._open().close() - continue - self._message_handler(message[:-1]) - - def _open(self): - return open(self._pipe_path, 'rb') - - def stop(self): - while self.is_alive(): - self._unblock() - self.join() - - def _unblock(self): - with suppress(OSError): - fd = osopen(self._pipe_path, O_WRONLY | O_NONBLOCK) - write(fd, self.stop_sequence) - close(fd) diff --git a/tfw/components/pipe_io/pipe_io_server/pipe_writer_server.py b/tfw/components/pipe_io/pipe_io_server/pipe_writer_server.py deleted file mode 100644 index 1161917..0000000 --- a/tfw/components/pipe_io/pipe_io_server/pipe_writer_server.py +++ /dev/null @@ -1,38 +0,0 @@ -from .pipe import Pipe, DEFAULT_PERMISSIONS -from .pipe_io_thread import PipeIOThread -from .pipe_writer_thread import PipeWriterThread - - -class PipeWriterServer(PipeIOThread): - def __init__( - self, - out_pipe, - permissions=DEFAULT_PERMISSIONS, - manage_pipes=True, - **kwargs - ): - super().__init__(**kwargs) - self._writer_thread = None - self._manage_pipes = manage_pipes - self._out_pipe = out_pipe - if self._manage_pipes: - Pipe(self.out_pipe).recreate(permissions) - - @property - def out_pipe(self): - return self._out_pipe - - def _io_threads(self): - self._writer_thread = PipeWriterThread( - self.out_pipe, - self._stop_event - ) - yield self._writer_thread - - def send_message(self, message): - self._writer_thread.write(message) - - def stop(self): - super().stop() - if self._manage_pipes: - Pipe(self.out_pipe).remove() diff --git a/tfw/components/pipe_io/pipe_io_server/pipe_writer_thread.py b/tfw/components/pipe_io/pipe_io_server/pipe_writer_thread.py deleted file mode 100644 index 3bc3fe7..0000000 --- a/tfw/components/pipe_io/pipe_io_server/pipe_writer_thread.py +++ /dev/null @@ -1,50 +0,0 @@ -from contextlib import suppress -from os import O_NONBLOCK, O_RDONLY, close -from os import open as osopen -from threading import Thread - -from .terminate_process_on_failure import terminate_process_on_failure -from .deque import Deque - - -class PipeWriterThread(Thread): - def __init__(self, pipe_path, stop_event): - super().__init__(daemon=True) - self._pipe_path = pipe_path - self._stop_event = stop_event - self._write_queue = Deque() - - def write(self, message): - self._write_queue.push(message) - - @terminate_process_on_failure - def run(self): - with self._open() as pipe: - while True: - message = self._write_queue.pop() - if message is None: - self._stop_event.set() - break - try: - pipe.write(message + b'\n') - pipe.flush() - except BrokenPipeError: - try: # pipe was reopened, close() flushed the message - pipe.close() - except BrokenPipeError: # close() discarded the message - self._write_queue.push_front(message) - pipe = self._open() - - def _open(self): - return open(self._pipe_path, 'wb') - - def stop(self): - while self.is_alive(): - self._unblock() - self.join() - - def _unblock(self): - with suppress(OSError): - fd = osopen(self._pipe_path, O_RDONLY | O_NONBLOCK) - self._write_queue.push_front(None) - close(fd) diff --git a/tfw/components/pipe_io/pipe_io_server/terminate_process_on_failure.py b/tfw/components/pipe_io/pipe_io_server/terminate_process_on_failure.py deleted file mode 100644 index 7a0804c..0000000 --- a/tfw/components/pipe_io/pipe_io_server/terminate_process_on_failure.py +++ /dev/null @@ -1,15 +0,0 @@ -from functools import wraps -from os import kill, getpid -from signal import SIGTERM -from traceback import print_exc - - -def terminate_process_on_failure(fun): - @wraps(fun) - def wrapper(*args, **kwargs): - try: - return fun(*args, **kwargs) - except: # pylint: disable=bare-except - print_exc() - kill(getpid(), SIGTERM) - return wrapper diff --git a/tfw/components/pipe_io/pipe_io_server/test_pipe_io_server.py b/tfw/components/pipe_io/pipe_io_server/test_pipe_io_server.py deleted file mode 100644 index 6a08847..0000000 --- a/tfw/components/pipe_io/pipe_io_server/test_pipe_io_server.py +++ /dev/null @@ -1,187 +0,0 @@ -# pylint: disable=redefined-outer-name -from os import stat, urandom -from os.path import exists, dirname, realpath, join -from stat import S_ISFIFO -from secrets import token_urlsafe -from random import randint, getrandbits, uniform -from threading import Thread -from json import dumps, loads - -import pytest - -from .pipe_io_server import PipeIOServer - - -class EchoPipeIOServer(PipeIOServer): - def handle_message(self, message): - self.send_message(message) - - -@pytest.fixture -def io_pipes(): - with EchoPipeIOServer(*get_test_init_params()) as pipe_io: - with IOPipes(pipe_io.in_pipe, pipe_io.out_pipe) as io_pipes: - yield io_pipes - - -def get_test_init_params(): - here = dirname(realpath(__file__)) - return join(here, 'in_pipe_tests'), join(here, 'out_pipe_tests') - - -def raise_if_thread_blocks(*, target, unblock_function): - thread = Thread(target=target) - thread.start() - unblock_function() - thread.join(timeout=1) - if thread.is_alive(): - raise RuntimeError('PipeIOServer failed to shut down!') - - -class IOPipes: - def __init__(self, in_pipe_path, out_pipe_path): - self.in_pipe_path = in_pipe_path - self.out_pipe_path = out_pipe_path - - def __enter__(self): - # pylint: disable=attribute-defined-outside-init - self.in_pipe = open(self.in_pipe_path, 'wb') - self.out_pipe = open(self.out_pipe_path, 'rb') - return self - - def __exit__(self, type_, value, traceback): - self.close() - - def close(self): - self.in_pipe.close() - self.out_pipe.close() - - def send_message(self, message): - self.in_pipe.write(message + b'\n') - self.in_pipe.flush() - - def recv(self): - return self.out_pipe.readline().rstrip(b'\n') - - -def pipes_exist(*paths): - predicate = lambda path: exists(path) and S_ISFIFO(stat(path).st_mode) - return all(predicate(path) for path in paths) - - -def test_manage_pipes(): - pipe_io = PipeIOServer(*get_test_init_params(), manage_pipes=True) - assert pipes_exist(pipe_io.in_pipe, pipe_io.out_pipe) - pipe_io.stop() - assert not pipes_exist(pipe_io.in_pipe, pipe_io.out_pipe) - - -def test_no_manage_pipes(): - pipe_io = PipeIOServer(*get_test_init_params(), manage_pipes=False) - assert not pipes_exist(pipe_io.in_pipe, pipe_io.out_pipe) - pipe_io.stop() - assert not pipes_exist(pipe_io.in_pipe, pipe_io.out_pipe) - - -def test_stop(): - pipe_io = EchoPipeIOServer(*get_test_init_params()) - pipe_io.start() - raise_if_thread_blocks(target=pipe_io.wait, unblock_function=pipe_io.stop) - - pipe_io = EchoPipeIOServer(*get_test_init_params()) - pipe_io.start() - with IOPipes(pipe_io.in_pipe, pipe_io.out_pipe) as iopipes: - raise_if_thread_blocks(target=pipe_io.wait, unblock_function=pipe_io.stop) - - pipe_io = EchoPipeIOServer(*get_test_init_params()) - pipe_io.start() - with IOPipes(pipe_io.in_pipe, pipe_io.out_pipe) as iopipes: - test_message = token_urlsafe(randint(128, 256)) - iopipes.send_message(test_message.encode()) - assert test_message == iopipes.recv().decode() - iopipes.send_message(test_message.encode()) - raise_if_thread_blocks(target=pipe_io.wait, unblock_function=pipe_io.stop) - - -@pytest.mark.parametrize( - 'test_data', [ - 'Cats and cheese', - 'You ever wonder why we are here?', - 'Lorem ipsum dolor sit amet', - 'You always have a plan, Dutch!', - ] -) -def test_io(io_pipes, test_data): - io_pipes.send_message(test_data.encode()) - assert io_pipes.recv().decode() == test_data - - -def test_io_random(io_pipes): - test_data = token_urlsafe(512) - for _ in range(100): - io_pipes.send_message(test_data.encode()) - assert io_pipes.recv().decode() == test_data - -@pytest.mark.parametrize( - 'test_data_size', [ - 1024, - 1024*1024, - 2*1024*1024, - 4*1024*1024, - 8*1024*1024, - 16*1024*1024, - 32*1024*1024 - ] -) -def test_io_large_data(io_pipes, test_data_size): - test_data = urandom(test_data_size).replace(b'\n', b'') - io_pipes.send_message(test_data) - received_data = io_pipes.recv() - assert received_data == test_data - - -def test_io_stress(io_pipes): - for _ in range(2222): - test_data = urandom(randint(1, 1024)).replace(b'\n', b'') - io_pipes.send_message(test_data) - assert io_pipes.recv() == test_data - - -def test_io_newlines(io_pipes): - times = randint(1, 512) - io_pipes.send_message(b'\n' * times) - for _ in range(times + 1): # IOPipes.send appends +1 - assert io_pipes.recv() == b'' - - -def test_json_io(io_pipes): - for _ in range(10): - test_data = { - f'{token_urlsafe(8)}': randint(1, 2 ** 20), - f'{token_urlsafe(9)}': [randint(1, 2 **10) for i in range(10)], - f'{token_urlsafe(4)}': f'{token_urlsafe(8)}\\\n{token_urlsafe(8)}\n{randint(1, 2 ** 10)}', - f'{token_urlsafe(11)}': { - f'{token_urlsafe(8)}': '', - f'{token_urlsafe(3)}': f'{token_urlsafe(8)}\n{token_urlsafe(8)}\n\\n{token_urlsafe(8)}', - f'{token_urlsafe(44)}': f'{token_urlsafe(8)}\n{token_urlsafe(8)} {token_urlsafe(8)}', - f'{token_urlsafe(6)}\n{token_urlsafe(4)}': bool(getrandbits(1)), - f'{token_urlsafe(8)}': None, - f'{token_urlsafe(21)} {token_urlsafe(4)}': None, - f'{token_urlsafe(3)}': uniform(randint(1, 100), randint(1, 100)), - f'{token_urlsafe(8)}': [token_urlsafe(4) for i in range(10)], - } - } - io_pipes.send_message(dumps(test_data).encode()) - assert loads(io_pipes.recv()) == test_data - - -def test_assign_message_handler(): - pipe_io = PipeIOServer(*get_test_init_params()) - pipe_io.handle_message = lambda msg: pipe_io.send_message(msg * 2) - pipe_io.start() - with IOPipes(pipe_io.in_pipe, pipe_io.out_pipe) as io_pipes: - for _ in range(100): - test_data = token_urlsafe(32).encode() - io_pipes.send_message(test_data) - assert io_pipes.recv() == test_data * 2 - pipe_io.stop() From 1210a30c31c28d009ebfe8d34a1af535ba1b7868 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 7 Aug 2019 09:42:03 +0200 Subject: [PATCH 104/174] Fix event order problem --- tfw/internals/inotify/test_inotify.py | 40 +++++++++++++++------------ 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/tfw/internals/inotify/test_inotify.py b/tfw/internals/inotify/test_inotify.py index d132706..3fabf6d 100644 --- a/tfw/internals/inotify/test_inotify.py +++ b/tfw/internals/inotify/test_inotify.py @@ -55,12 +55,14 @@ class InotifyContext: def join(self, path): return join(self.workdir, path) - def check_event(self, event_type, path): - self.missing_events += 1 - event = self.event_to_queue[event_type].get(timeout=0.1) - assert isinstance(event, event_type) - assert event.src_path == path - return event + def check_event(self, event_type, paths): + if isinstance(paths, str): + paths = [paths] + self.missing_events += len(paths) + for _ in range(len(paths)): + event = self.event_to_queue[event_type].get(timeout=0.1) + assert isinstance(event, event_type) + assert event.src_path in paths def check_empty(self, event_type): with pytest.raises(Empty): @@ -69,7 +71,7 @@ class InotifyContext: def check_any(self): attrs = self.observer.__dict__.values() total = sum([q.qsize() for q in attrs if isinstance(q, Queue)]) - return total+self.missing_events == len(self.observer.any_list) + return total + self.missing_events == len(self.observer.any_list) class InotifyTestObserver(InotifyObserver): @@ -100,7 +102,7 @@ def generate_name(): def context(): with TemporaryDirectory() as workdir: subdir = join(workdir, generate_name()) - subfile = join(subdir, generate_name()+'.txt') + subfile = join(subdir, generate_name() + '.txt') mkdir(subdir) Path(subfile).touch() monitor = InotifyTestObserver(workdir, recursive=True) @@ -124,16 +126,21 @@ def test_modify(context): context.missing_events += 1 except Empty: break - rename(context.subfile, context.subfile+'_new') + rename(context.subfile, context.subfile + '_new') context.check_event(InotifyDirModifiedEvent, context.subdir) assert context.check_any() -def test_move(context): - rename(context.subdir, context.subdir+'_new') - context.check_event(InotifyDirMovedEvent, context.subdir) +def test_file_move(context): + rename(context.subfile, context.subfile + '_new') context.check_event(InotifyFileMovedEvent, context.subfile) assert context.check_any() +def test_folder_move(context): + remove(context.subfile) + rename(context.subdir, context.subdir + '_new') + context.check_event(InotifyDirMovedEvent, context.subdir) + assert context.check_any() + def test_delete(context): rmtree(context.subdir) context.check_event(InotifyFileDeletedEvent, context.subfile) @@ -171,9 +178,8 @@ def test_exclude(context): context.observer.exclude = None def test_stress(context): - newfile = [] - for i in range(1024): - newfile.append(context.create_random_file(context.subdir, '.txt')) - for i in range(1024): - context.check_event(InotifyFileCreatedEvent, newfile[i]) + newfiles = [] + for _ in range(1024): + newfiles.append(context.create_random_file(context.subdir, '.txt')) + context.check_event(InotifyFileCreatedEvent, newfiles) assert context.check_any() From d31a850a4e70cce27b328e2c172d84b9c8665f14 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 7 Aug 2019 09:44:03 +0200 Subject: [PATCH 105/174] Simplify IDE handler and file manager --- .../ide/file_manager/file_manager.py | 123 +++++--------- .../ide/file_manager/test_file_manager.py | 147 ++++++----------- tfw/components/ide/ide_handler.py | 156 ++++-------------- 3 files changed, 127 insertions(+), 299 deletions(-) diff --git a/tfw/components/ide/file_manager/file_manager.py b/tfw/components/ide/file_manager/file_manager.py index 0e4fc6a..e416cb9 100644 --- a/tfw/components/ide/file_manager/file_manager.py +++ b/tfw/components/ide/file_manager/file_manager.py @@ -1,93 +1,56 @@ -from typing import Iterable +from functools import wraps from glob import glob from fnmatch import fnmatchcase -from os.path import basename, isfile, join, relpath, exists, isdir, realpath +from os.path import dirname, isdir, isfile, realpath -class FileManager: # pylint: disable=too-many-instance-attributes - def __init__(self, working_directory, allowed_directories, selected_file=None, exclude=None): - self._exclude, self.exclude = [], exclude - self._allowed_directories, self.allowed_directories = None, allowed_directories - self._workdir, self.workdir = None, working_directory - self._filename, self.filename = None, selected_file or self.files[0] +def _with_is_allowed(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + if self._is_allowed(args[0]): # pylint: disable=protected-access + return func(self, *args, **kwargs) + raise ValueError('Forbidden path.') + return wrapper - @property - def exclude(self): - return self._exclude - @exclude.setter - def exclude(self, exclude): - if exclude is None: - return - if not isinstance(exclude, Iterable): - raise TypeError('Exclude must be Iterable!') - self._exclude = exclude - - @property - def workdir(self): - return self._workdir - - @workdir.setter - def workdir(self, directory): - if not exists(directory) or not isdir(directory): - raise EnvironmentError(f'"{directory}" is not a directory!') - if not self._is_in_allowed_dir(directory): - raise EnvironmentError(f'Directory "{directory}" is not allowed!') - self._workdir = directory - - @property - def allowed_directories(self): - return self._allowed_directories - - @allowed_directories.setter - def allowed_directories(self, directories): - self._allowed_directories = [realpath(directory) for directory in directories] - - @property - def filename(self): - return self._filename - - @filename.setter - def filename(self, filename): - if filename not in self.files: - raise EnvironmentError('No such file in workdir!') - self._filename = filename +class FileManager: # pylint: disable=too-many-instance-attributes + def __init__(self, patterns): + self.patterns = patterns @property def files(self): - return [ - self._relpath(file) - for file in glob(join(self._workdir, '**/*'), recursive=True) - if isfile(file) - and self._is_in_allowed_dir(file) - and not self._is_blacklisted(file) - ] + return list(set( + path + for pattern in self.patterns + for path in glob(pattern, recursive=True) + if isfile(path) and self._is_allowed(path) + )) @property - def file_contents(self): - with open(self._filepath(self.filename), 'rb', buffering=0) as ifile: + def parents(self): + return list(set( + self._find_directory(pattern) + for pattern in self.patterns + )) + + @staticmethod + def _find_directory(pattern): + while pattern and not isdir(pattern): + pattern = dirname(pattern) + return pattern + + def _is_allowed(self, filepath): + return any( + fnmatchcase(realpath(filepath), pattern) + for pattern in self.patterns + ) + + @_with_is_allowed + def read_file(self, filepath): # pylint: disable=no-self-use + with open(filepath, 'rb', buffering=0) as ifile: return ifile.read().decode(errors='surrogateescape') - @file_contents.setter - def file_contents(self, value): - with open(self._filepath(self.filename), 'wb', buffering=0) as ofile: - ofile.write(value.encode()) - - def _is_in_allowed_dir(self, path): - return any( - realpath(path).startswith(allowed_dir) - for allowed_dir in self.allowed_directories - ) - - def _is_blacklisted(self, file): - return any( - fnmatchcase(file, blacklisted) or - fnmatchcase(basename(file), blacklisted) - for blacklisted in self.exclude - ) - - def _filepath(self, filename): - return join(self._workdir, filename) - - def _relpath(self, filename): - return relpath(self._filepath(filename), start=self._workdir) + @_with_is_allowed + def write_file(self, filepath, contents): # pylint: disable=no-self-use + with open(filepath, 'wb', buffering=0) as ofile: + ofile.write(contents.encode()) diff --git a/tfw/components/ide/file_manager/test_file_manager.py b/tfw/components/ide/file_manager/test_file_manager.py index 8f978d3..08e4f25 100644 --- a/tfw/components/ide/file_manager/test_file_manager.py +++ b/tfw/components/ide/file_manager/test_file_manager.py @@ -1,8 +1,8 @@ # pylint: disable=redefined-outer-name from dataclasses import dataclass from secrets import token_urlsafe +from os import mkdir, symlink from os.path import join -from os import chdir, mkdir, symlink from pathlib import Path from tempfile import TemporaryDirectory @@ -13,112 +13,75 @@ from .file_manager import FileManager @dataclass class ManagerContext: - folder: str + workdir: str + subdir: str + subfile: str manager: FileManager - def join(self, path): - return join(self.folder, path) + def create_random_file(self, dirname, extension): + filename = self.join(f'{dirname}/{generate_name()}{extension}') + Path(filename).touch() + return filename + def create_random_folder(self, basepath): + dirname = self.join(f'{basepath}/{generate_name()}') + mkdir(dirname) + return dirname + + def create_random_link(self, source, dirname, extension): + linkname = self.join(f'{dirname}/{generate_name()}{extension}') + symlink(source, linkname) + return linkname + + def join(self, path): + return join(self.workdir, path) + +def generate_name(): + return token_urlsafe(16) @pytest.fixture() def context(): - dirs = {} - with TemporaryDirectory() as workdir: - chdir(workdir) - for name in ['allowed', 'excluded', 'invis']: - node = join(workdir, name) - mkdir(node) - Path(join(node, 'empty.txt')).touch() - Path(join(node, 'empty.bin')).touch() - dirs[name] = node + subdir = join(workdir, generate_name()) + subfile = join(subdir, generate_name() + '.txt') + mkdir(subdir) + Path(subfile).touch() + manager = FileManager([join(workdir, '**/*.txt')]) + yield ManagerContext(workdir, subdir, subfile, manager) - yield ManagerContext( - workdir, - FileManager( - dirs['allowed'], - [dirs['allowed'], dirs['excluded']], - exclude=['*/excluded/*'] - ) - ) +def test_matching_files(context): + newdir = context.create_random_folder(context.subdir) + newfile = context.create_random_file(newdir, '.txt') + newlink = context.create_random_link(newfile, newdir, '.txt') + assert set(context.manager.files) == {context.subfile, newfile, newlink} -@pytest.mark.parametrize('subdir', ['allowed/', 'excluded/']) -def test_select_allowed_dirs(context, subdir): - context.manager.workdir = context.join(subdir) - assert context.manager.workdir == context.join(subdir) - newdir = context.join(subdir+'deep') - mkdir(newdir) - context.manager.workdir = newdir - assert context.manager.workdir == newdir +def test_unmatching_files(context): + newtxt = context.create_random_file(context.workdir, '.txt') + newbin = context.create_random_file(context.subdir, '.bin') + context.create_random_link(newtxt, context.subdir, '.txt') + context.create_random_link(newbin, context.subdir, '.txt') + assert context.manager.files == [context.subfile] -@pytest.mark.parametrize('invdir', ['', 'invis']) -def test_select_forbidden_dirs(context, invdir): - fullpath = context.join(invdir) - with pytest.raises(OSError): - context.manager.workdir = fullpath - assert context.manager.workdir != fullpath - context.manager.allowed_directories += [fullpath] - context.manager.workdir = fullpath - assert context.manager.workdir == fullpath - - -@pytest.mark.parametrize('filename', ['another.txt', '*.txt']) -def test_select_allowed_files(context, filename): - Path(context.join('allowed/'+filename)).touch() - assert filename in context.manager.files - context.manager.filename = filename - assert context.manager.filename == filename - -@pytest.mark.parametrize('path', [ - {'dir': 'allowed/', 'file': 'illegal.bin'}, - {'dir': 'excluded/', 'file': 'legal.txt'}, - {'dir': 'allowed/', 'file': token_urlsafe(16)+'.bin'}, - {'dir': 'excluded/', 'file': token_urlsafe(16)+'.txt'}, - {'dir': 'allowed/', 'file': token_urlsafe(32)+'.bin'}, - {'dir': 'excluded/', 'file': token_urlsafe(32)+'.txt'} -]) -def test_select_excluded_files(context, path): - context.manager.workdir = context.join(path['dir']) - context.manager.exclude = ['*/excluded/*', '*.bin'] - Path(context.join(path['dir']+path['file'])).touch() - assert path['file'] not in context.manager.files - with pytest.raises(OSError): - context.manager.filename = path['file'] - -@pytest.mark.parametrize('path', [ - {'src': 'excluded/empty.txt', 'dst': 'allowed/link.txt'}, - {'src': 'invis/empty.txt', 'dst': 'allowed/link.txt'}, - {'src': 'excluded/empty.txt', 'dst': 'allowed/'+token_urlsafe(16)+'.txt'}, - {'src': 'invis/empty.txt', 'dst': 'allowed/'+token_urlsafe(16)+'.txt'}, - {'src': 'excluded/empty.txt', 'dst': 'allowed/'+token_urlsafe(32)+'.txt'}, - {'src': 'invis/empty.txt', 'dst': 'allowed/'+token_urlsafe(32)+'.txt'} -]) -def test_select_excluded_symlinks(context, path): - symlink(context.join(path['src']), context.join(path['dst'])) - assert path['dst'] not in context.manager.files +def test_parents(context): + newdir = context.create_random_folder(context.workdir) + context.manager.patterns += [f'{newdir}/[!/@]*/**/?.c'] + assert set(context.manager.parents) == {context.workdir, newdir} def test_read_write_file(context): for _ in range(128): - context.manager.filename = 'empty.txt' content = token_urlsafe(32) - context.manager.file_contents = content - assert context.manager.file_contents == content - with open(context.join('allowed/empty.txt'), 'r') as ifile: + context.manager.write_file(context.subfile, content) + assert context.manager.read_file(context.subfile) == content + with open(context.subfile, 'r') as ifile: assert ifile.read() == content def test_regular_ide_actions(context): - context.manager.workdir = context.join('allowed') - newfile1, newfile2 = token_urlsafe(16), token_urlsafe(16) - Path(context.join(f'allowed/{newfile1}')).touch() - Path(context.join(f'allowed/{newfile2}')).touch() - for _ in range(8): + newfile1 = context.create_random_file(context.subdir, '.txt') + newfile2 = context.create_random_file(context.subdir, '.txt') + for _ in range(4): context.manager.filename = newfile1 - content1 = token_urlsafe(32) - context.manager.file_contents = content1 - context.manager.filename = newfile2 - content2 = token_urlsafe(32) - context.manager.file_contents = content2 - context.manager.filename = newfile1 - assert context.manager.file_contents == content1 - context.manager.filename = newfile2 - assert context.manager.file_contents == content2 + content1, content2 = token_urlsafe(32), token_urlsafe(32) + context.manager.write_file(newfile1, content1) + context.manager.write_file(newfile2, content2) + assert context.manager.read_file(newfile1) == content1 + assert context.manager.read_file(newfile2) == content2 diff --git a/tfw/components/ide/ide_handler.py b/tfw/components/ide/ide_handler.py index 38f7842..f7435d6 100644 --- a/tfw/components/ide/ide_handler.py +++ b/tfw/components/ide/ide_handler.py @@ -1,4 +1,5 @@ import logging +from os.path import isfile from tfw.internals.networking import Scope from tfw.internals.inotify import InotifyObserver @@ -32,161 +33,62 @@ BUILD_ARTIFACTS = ( class IdeHandler: - keys = ['ide'] - # pylint: disable=too-many-arguments,anomalous-backslash-in-string - """ - Event handler implementing the backend of our browser based IDE. - By default all files in the directory specified in __init__ are displayed - on the fontend. Note that this is a stateful component. + keys = ['ide.read', 'ide.write'] - When any file in the selected directory changes they are automatically refreshed - on the frontend (this is done by listening to inotify events). - - This EventHandler accepts messages that have a data['command'] key specifying - a command to be executed. - - The API of each command is documented in their respective handler. - """ - def __init__(self, *, directory, allowed_directories, selected_file=None, exclude=None): - """ - :param key: the key this instance should listen to - :param directory: working directory which the EventHandler should serve files from - :param allowed_directories: list of directories that can be switched to using selectdir - :param selected_file: file that is selected by default - :param exclude: list of filenames that should not appear between files (for .o, .pyc, etc.) - """ + def __init__(self, *, patterns, initial_file=''): self.connector = None - try: - self.filemanager = FileManager( - allowed_directories=allowed_directories, - working_directory=directory, - selected_file=selected_file, - exclude=exclude - ) - except IndexError: - raise EnvironmentError( - f'No file(s) in IdeEventHandler working_directory "{directory}"!' - ) + self.filemanager = FileManager(patterns) + self._initial_file = initial_file self.monitor = InotifyObserver( - self.filemanager.allowed_directories, + path=self.filemanager.parents, exclude=BUILD_ARTIFACTS ) self.monitor.on_modified = self._reload_frontend self.monitor.start() self.commands = { - 'read': self.read, - 'write': self.write, - 'select': self.select, - 'selectdir': self.select_dir, - 'exclude': self.exclude + 'ide.read' : self.read, + 'ide.write' : self.write } + @property + def initial_file(self): + if not isfile(self._initial_file): + self._initial_file = self.filemanager.files[0] + return self._initial_file + def _reload_frontend(self, event): # pylint: disable=unused-argument - self.send_message({ - 'key': 'ide', - 'data': {'command': 'reload'} - }) + self.send_message({'key': 'ide.reload'}) def send_message(self, message): self.connector.send_message(message, scope=Scope.WEBSOCKET) - def read(self, data): - """ - Read the currently selected file. - - :return dict: TFW message data containing key 'content' - (contents of the selected file) - """ + def read(self, message): + if message.get('files'): + self.filemanager.patterns = message['files'] try: - data['content'] = self.filemanager.file_contents + message['content'] = self.filemanager.read_file(message['filename']) except PermissionError: - data['content'] = 'You have no permission to open that file :(' + message['content'] = 'You have no permission to open that file :(' except FileNotFoundError: - data['content'] = 'This file was removed :(' + message['content'] = 'This file was removed :(' except Exception: # pylint: disable=broad-except - data['content'] = 'Failed to read file :(' - return data + message['content'] = 'Failed to read file :(' - def write(self, data): - """ - Overwrites a file with the desired string. - - :param data: TFW message data containing key 'content' - (new file content) - - """ + def write(self, message): try: - self.filemanager.file_contents = data['content'] + self.filemanager.write_file(message['filename'], message['content']) except Exception: # pylint: disable=broad-except LOG.exception('Error writing file!') - del data['content'] - return data - - def select(self, data): - """ - Selects a file from the current directory. - - :param data: TFW message data containing 'filename' - (name of file to select relative to the current directory) - """ - try: - self.filemanager.filename = data['filename'] - except EnvironmentError: - LOG.exception('Failed to select file "%s"', data['filename']) - return data - - def select_dir(self, data): - """ - Select a new working directory to display files from. - - :param data: TFW message data containing 'directory' - (absolute path of diretory to select. - must be a path whitelisted in - self.allowed_directories) - """ - try: - self.filemanager.workdir = data['directory'] - try: - self.filemanager.filename = self.filemanager.files[0] - self.read(data) - except IndexError: - data['content'] = 'No files in this directory :(' - except EnvironmentError as err: - LOG.error( - 'Failed to select directory "%s". Reason: %s', - data['directory'], str(err) - ) - return data - - def exclude(self, data): - """ - Overwrite list of excluded files - - :param data: TFW message data containing 'exclude' - (list of unix-style filename patterns to be excluded, - e.g.: ["\*.pyc", "\*.o") - """ - try: - self.filemanager.exclude = list(data['exclude']) - except TypeError: - LOG.error('Exclude must be Iterable!') - return data - - def attach_fileinfo(self, data): - """ - Basic information included in every response to the frontend. - """ - data['filename'] = self.filemanager.filename - data['files'] = self.filemanager.files - data['directory'] = self.filemanager.workdir + del message['content'] def handle_event(self, message, _): try: - data = message['data'] - message['data'] = self.commands[data['command']](data) - self.attach_fileinfo(data) + if message['filename'] == '': + message['filename'] = self.initial_file + self.commands[message['key']](message) + message['files'] = self.filemanager.files self.send_message(message) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) From dded7fd65cee74d3841a2fdbb95f66473d645194 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 7 Aug 2019 09:46:58 +0200 Subject: [PATCH 106/174] Update FSM related classes according to the new API --- tfw/components/fsm/fsm_handler.py | 41 ++++------------------- tfw/components/fsm/fsm_updater.py | 2 +- tfw/internals/event_handling/fsm_aware.py | 4 +-- 3 files changed, 10 insertions(+), 37 deletions(-) diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index 60b8c16..61f901e 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -1,6 +1,6 @@ import logging -from tfw.internals.crypto import KeyManager, sign_message, verify_message +from tfw.internals.crypto import KeyManager, sign_message from tfw.internals.networking import Scope from .fsm_updater import FSMUpdater @@ -9,57 +9,30 @@ LOG = logging.getLogger(__name__) class FSMHandler: - keys = ['fsm'] - """ - EventHandler responsible for managing the state machine of - the framework (TFW FSM). - - tfw.networking.TFWServer instances automatically send 'trigger' - commands to the event handler listening on the 'fsm' key, - which should be an instance of this event handler. - - This event handler accepts messages that have a - data['command'] key specifying a command to be executed. - - An 'fsm_update' message is broadcasted after every successful - command. - """ + keys = ['fsm.step', 'fsm.update'] def __init__(self, *, fsm_type): self.fsm = fsm_type() self._fsm_updater = FSMUpdater(self.fsm) self.auth_key = KeyManager().auth_key self.command_handlers = { - 'trigger': self.handle_trigger, - 'update': self.handle_update + 'fsm.step' : self.handle_step, + 'fsm.update' : self.handle_update } def handle_event(self, message, connector): try: - message = self.command_handlers[message['data']['command']](message) + message = self.command_handlers[message['key']](message) if message: fsm_update_message = self._fsm_updater.fsm_update - sign_message(self.auth_key, message) sign_message(self.auth_key, fsm_update_message) connector.send_message(fsm_update_message, Scope.BROADCAST) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) - def handle_trigger(self, message): - """ - Attempts to step the FSM with the supplied trigger. - - :param message: TFW message with a data field containing - the action to try triggering in data['value'] - """ - trigger = message['data']['value'] - if self.fsm.step(trigger): - return message - return None + def handle_step(self, message): + return message if self.fsm.step(message['trigger']) else None def handle_update(self, message): - """ - Does nothing, but triggers an 'fsm_update' message. - """ # pylint: disable=no-self-use return message diff --git a/tfw/components/fsm/fsm_updater.py b/tfw/components/fsm/fsm_updater.py index 93f9e7a..d1b42ec 100644 --- a/tfw/components/fsm/fsm_updater.py +++ b/tfw/components/fsm/fsm_updater.py @@ -5,7 +5,7 @@ class FSMUpdater: @property def fsm_update(self): return { - 'key': 'fsm_update', + 'key': 'fsm.announce', **self.fsm_update_data } diff --git a/tfw/internals/event_handling/fsm_aware.py b/tfw/internals/event_handling/fsm_aware.py index 5254e68..b700dba 100644 --- a/tfw/internals/event_handling/fsm_aware.py +++ b/tfw/internals/event_handling/fsm_aware.py @@ -6,7 +6,7 @@ LOG = logging.getLogger(__name__) class FSMAware: - keys = ['fsm_update'] + keys = ['fsm.announce'] """ Base class for stuff that has to be aware of the framework FSM. This is done by processing 'fsm_update' messages. @@ -18,7 +18,7 @@ class FSMAware: self._auth_key = KeyManager().auth_key def process_message(self, message): - if message['key'] == 'fsm_update': + if message['key'] == 'fsm.announce': if verify_message(self._auth_key, message): self._handle_fsm_update(message) From e414ea2631e946bcfb690218dd4fb9fa04ef7cec Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 7 Aug 2019 09:47:28 +0200 Subject: [PATCH 107/174] Simplify terminal handler --- tfw/components/terminal/terminal_handler.py | 55 ++++----------------- 1 file changed, 9 insertions(+), 46 deletions(-) diff --git a/tfw/components/terminal/terminal_handler.py b/tfw/components/terminal/terminal_handler.py index c8df229..7d3e041 100644 --- a/tfw/components/terminal/terminal_handler.py +++ b/tfw/components/terminal/terminal_handler.py @@ -7,40 +7,25 @@ LOG = logging.getLogger(__name__) class TerminalHandler: - keys = ['shell'] - """ - Event handler responsible for managing terminal sessions for frontend xterm - sessions to connect to. You need to instanciate this in order for frontend - terminals to work. + keys = ['terminal.write'] - This EventHandler accepts messages that have a data['command'] key specifying - a command to be executed. - The API of each command is documented in their respective handler. - """ - def __init__(self, *, port, user, workind_directory, histfile): - """ - :param key: key this EventHandler listens to - :param monitor: tfw.components.HistoryMonitor instance to read command history from - """ - self.connector = None + def __init__(self, *, port, user, working_directory, histfile): + self.connector, self._historymonitor = None, None self._histfile = histfile - self._historymonitor = None bash_as_user_cmd = ['sudo', '-u', user, 'bash'] self.terminado_server = TerminadoMiniServer( '/terminal', port, - workind_directory, + working_directory, bash_as_user_cmd ) + self.terminado_server.listen() self.commands = { - 'write': self.write, - 'read': self.read + 'terminal.write': self.handle_write } - self.terminado_server.listen() - def start(self): self._historymonitor = BashMonitor(self.connector, self._histfile) self._historymonitor.start() @@ -51,34 +36,12 @@ class TerminalHandler: def handle_event(self, message, _): try: - data = message['data'] - message['data'] = self.commands[data['command']](data) + self.commands[message['key']](message) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) - def write(self, data): - """ - Writes a string to the terminal session (on the pty level). - Useful for pre-typing and executing commands for the user. - - :param data: TFW message data containing 'value' - (command to be written to the pty) - """ - self.terminado_server.pty.write(data['value']) - return data - - def read(self, data): - """ - Reads the history of commands executed. - - :param data: TFW message data containing 'count' - (the number of history elements to return) - :return dict: message with list of commands in data['history'] - """ - data['count'] = int(data.get('count', 1)) - if self.historymonitor: - data['history'] = self.historymonitor.history[-data['count']:] - return data + def handle_write(self, message): + self.terminado_server.pty.write(message['command']) def cleanup(self): self.terminado_server.stop() From 4b7510e704096783de02b9c56b9c73f48c69ec07 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 7 Aug 2019 09:47:57 +0200 Subject: [PATCH 108/174] Update process- and logmonitor according to the new API --- .../log_inotify_observer.py | 11 ++--- .../process_management/process_handler.py | 34 +++++--------- .../process_management/process_log_handler.py | 45 +++++-------------- 3 files changed, 25 insertions(+), 65 deletions(-) diff --git a/tfw/components/process_management/log_inotify_observer.py b/tfw/components/process_management/log_inotify_observer.py index cbbd5dc..4ea88cf 100644 --- a/tfw/components/process_management/log_inotify_observer.py +++ b/tfw/components/process_management/log_inotify_observer.py @@ -7,7 +7,7 @@ from .supervisor import ProcessLogManager class LogInotifyObserver(InotifyObserver, ProcessLogManager): - def __init__(self, connector, supervisor_uri, process_name, log_tail=0): + def __init__(self, connector, process_name, supervisor_uri, log_tail=0): self._prevent_log_recursion() self._connector = connector self._process_name = process_name @@ -36,10 +36,7 @@ class LogInotifyObserver(InotifyObserver, ProcessLogManager): def on_modified(self, event): self._connector.send_message({ - 'key': 'processlog', - 'data': { - 'command': 'new_log', - 'stdout': self.read_stdout(self.process_name, tail=self.log_tail), - 'stderr': self.read_stderr(self.process_name, tail=self.log_tail) - } + 'key': 'log.new', + 'stdout': self.read_stdout(self.process_name, tail=self.log_tail), + 'stderr': self.read_stderr(self.process_name, tail=self.log_tail) }, Scope.BROADCAST) diff --git a/tfw/components/process_management/process_handler.py b/tfw/components/process_management/process_handler.py index 1c54118..80e1a4e 100644 --- a/tfw/components/process_management/process_handler.py +++ b/tfw/components/process_management/process_handler.py @@ -9,43 +9,31 @@ LOG = logging.getLogger(__name__) class ProcessHandler(ProcessManager, ProcessLogManager): - keys = ['processmanager'] - """ - Event handler that can manage processes managed by supervisor. - - This EventHandler accepts messages that have a data['command'] key specifying - a command to be executed. - Every message must contain a data['process_name'] field with the name of the - process to manage. This is the name specified in supervisor config files like so: - [program:someprogram] - - Commands available: start, stop, restart, readlog - (the names are as self-documenting as it gets) - """ + keys = ['process.start', 'process.stop', 'process.restart'] def __init__(self, *, supervisor_uri, log_tail=0): ProcessManager.__init__(self, supervisor_uri) ProcessLogManager.__init__(self, supervisor_uri) self.log_tail = log_tail + self.commands = { - 'start': self.start_process, - 'stop': self.stop_process, - 'restart': self.restart_process + 'process.start': self.start_process, + 'process.stop': self.stop_process, + 'process.restart': self.restart_process } def handle_event(self, message, connector): try: - data = message['data'] try: - self.commands[data['command']](data['process_name']) + self.commands[message['key']](message['name']) except SupervisorFault as fault: - message['data']['error'] = fault.faultString + message['error'] = fault.faultString finally: - message['data']['stdout'] = self.read_stdout( - data['process_name'], + message['stdout'] = self.read_stdout( + message['name'], self.log_tail ) - message['data']['stderr'] = self.read_stderr( - data['process_name'], + message['stderr'] = self.read_stderr( + message['name'], self.log_tail ) connector.send_message(message, scope=Scope.WEBSOCKET) diff --git a/tfw/components/process_management/process_log_handler.py b/tfw/components/process_management/process_log_handler.py index 8386120..b23874e 100644 --- a/tfw/components/process_management/process_log_handler.py +++ b/tfw/components/process_management/process_log_handler.py @@ -6,63 +6,38 @@ LOG = logging.getLogger(__name__) class ProcessLogHandler: - keys = ['logmonitor'] - """ - Monitors the output of a supervisor process (stdout, stderr) and - sends the results to the frontend. + keys = ['log.set'] - Accepts messages that have a data['command'] key specifying - a command to be executed. - - The API of each command is documented in their respective handler. - """ def __init__(self, *, process_name, supervisor_uri, log_tail=0): - self.connector = None + self.connector, self._monitor = None, None self.process_name = process_name self._supervisor_uri = supervisor_uri self._initial_log_tail = log_tail - self._monitor = None self.command_handlers = { - 'process_name': self.handle_process_name, - 'log_tail': self.handle_log_tail + 'log.set': self.handle_set } def start(self): self._monitor = LogInotifyObserver( connector=self.connector, - supervisor_uri=self._supervisor_uri, process_name=self.process_name, + supervisor_uri=self._supervisor_uri, log_tail=self._initial_log_tail ) self._monitor.start() def handle_event(self, message, _): try: - data = message['data'] - self.command_handlers[data['command']](data) + self.command_handlers[message['key']](message) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) - def handle_process_name(self, data): - """ - Changes the monitored process. - - :param data: TFW message data containing 'value' - (name of the process to monitor) - """ - self._monitor.process_name = data['value'] - - def handle_log_tail(self, data): - """ - Sets tail length of the log the monitor will send - to the frontend (the monitor will send back the last - 'value' characters of the log). - - :param data: TFW message data containing 'value' - (new tail length) - """ - self._monitor.log_tail = data['value'] + def handle_set(self, data): + if data.get('name'): + self._monitor.process_name = data['name'] + if data.get('tail'): + self._monitor.log_tail = data['tail'] def cleanup(self): self._monitor.stop() From f8ff0bcbb4435225f230fa4974f42683be363e3e Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 7 Aug 2019 09:49:05 +0200 Subject: [PATCH 109/174] Simplify sending bot messages --- tfw/components/frontend/message_sender.py | 58 ++++++++--------------- 1 file changed, 20 insertions(+), 38 deletions(-) diff --git a/tfw/components/frontend/message_sender.py b/tfw/components/frontend/message_sender.py index 72915c2..0f4f249 100644 --- a/tfw/components/frontend/message_sender.py +++ b/tfw/components/frontend/message_sender.py @@ -1,48 +1,30 @@ class MessageSender: - """ - Provides mechanisms to send messages to our frontend messaging component. - """ def __init__(self, uplink): self.uplink = uplink - self.key = 'message' - self.queue_key = 'queueMessages' - def send(self, originator, message): - """ - Sends a message. - :param originator: name of sender to be displayed on the frontend - :param message: message to send - """ + def send(self, message, originator=None): message = { - 'key': self.key, - 'data': { - 'originator': originator, - 'message': message - } + 'key': 'message.send', + 'message': message } + if originator: + message['originator'] = originator self.uplink.send_message(message) - def queue_messages(self, originator, messages): - """ - Queues a list of messages to be displayed in a chatbot-like manner. - :param originator: name of sender to be displayed on the frontend - :param messages: list of messages to queue - """ - message = { - 'key': self.queue_key, - 'data': { - 'messages': [ - {'message': message, 'originator': originator} - for message in messages - ] - } + def queue_messages(self, messages, originator=None): + message_queue = { + 'key': 'message.queue', + 'value': [] } - self.uplink.send_message(message) + for message in messages: + next_message = {'message': message} + if originator: + next_message['originator'] = originator + message_queue['value'].append(next_message) + self.uplink.send_message(message_queue) - @staticmethod - def generate_messages_from_queue(queue_message): - for message in queue_message['data']['messages']: - yield { - 'key': 'message', - 'data': message - } + def set_originator(self, originator): + self.uplink.send_message({ + 'key': 'message.config', + 'originator': originator + }) From 09ffe2bdcf4639697007056c88e51ee4a82c2800 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 7 Aug 2019 09:49:25 +0200 Subject: [PATCH 110/174] Forward relevant messages to the frontend --- tfw/components/frontend/frontend_handler.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tfw/components/frontend/frontend_handler.py b/tfw/components/frontend/frontend_handler.py index 5829ad7..7e35bc5 100644 --- a/tfw/components/frontend/frontend_handler.py +++ b/tfw/components/frontend/frontend_handler.py @@ -4,7 +4,21 @@ from .message_storage import FrontendMessageStorage class FrontendHandler: - keys = ['message', 'queueMessages', 'dashboard', 'console'] + # TODO: do not store dashboard messages like reloadFrontend + keys = [ + 'console.read', + 'console.write', + 'console.showLiveLogs', + 'console.rewriteContentWithProcessLogsOnDeploy', + 'dashboard.layout', + 'dashboard.hideMessages', + 'dashboard.terminalMenuItem', + 'dashboard.reloadFrontend', + 'dashboard.reloadIframe', + 'message.config', + 'message.queue', + 'message.send' + ] def __init__(self): self.connector = None From 13179e59ebb91141e63d8b6402b4acda0a92815f Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 7 Aug 2019 11:24:56 +0200 Subject: [PATCH 111/174] Restore generate_messages_from_queue() --- tfw/components/frontend/message_sender.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tfw/components/frontend/message_sender.py b/tfw/components/frontend/message_sender.py index 0f4f249..93826df 100644 --- a/tfw/components/frontend/message_sender.py +++ b/tfw/components/frontend/message_sender.py @@ -28,3 +28,11 @@ class MessageSender: 'key': 'message.config', 'originator': originator }) + + @staticmethod + def generate_messages_from_queue(queue_message): + for message in queue_message['value']: + yield { + 'key': 'message.send', + **message + } From f5e7d6016aff6d77b101c74a2be705d116663fc2 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 8 Aug 2019 08:27:41 +0200 Subject: [PATCH 112/174] Fix condition in PR #62 --- tfw/components/fsm/fsm_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index 61f901e..c9cb6eb 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -31,7 +31,8 @@ class FSMHandler: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) def handle_step(self, message): - return message if self.fsm.step(message['trigger']) else None + if self.fsm.step(message['trigger']): + return message def handle_update(self, message): # pylint: disable=no-self-use From 35e5b595d137f7bbed4f04f5f739bf3ec7624d90 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 8 Aug 2019 14:45:59 +0200 Subject: [PATCH 113/174] Rename message keys --- tfw/components/fsm/fsm_handler.py | 6 +++--- tfw/components/fsm/fsm_updater.py | 2 +- tfw/components/process_management/log_inotify_observer.py | 2 +- tfw/components/process_management/process_log_handler.py | 4 ++-- tfw/internals/event_handling/fsm_aware.py | 8 ++------ 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index c9cb6eb..5d28c1f 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -9,7 +9,7 @@ LOG = logging.getLogger(__name__) class FSMHandler: - keys = ['fsm.step', 'fsm.update'] + keys = ['fsm.step', 'fsm.announce'] def __init__(self, *, fsm_type): self.fsm = fsm_type() self._fsm_updater = FSMUpdater(self.fsm) @@ -17,7 +17,7 @@ class FSMHandler: self.command_handlers = { 'fsm.step' : self.handle_step, - 'fsm.update' : self.handle_update + 'fsm.announce' : self.handle_announce } def handle_event(self, message, connector): @@ -34,6 +34,6 @@ class FSMHandler: if self.fsm.step(message['trigger']): return message - def handle_update(self, message): + def handle_announce(self, message): # pylint: disable=no-self-use return message diff --git a/tfw/components/fsm/fsm_updater.py b/tfw/components/fsm/fsm_updater.py index d1b42ec..276b65d 100644 --- a/tfw/components/fsm/fsm_updater.py +++ b/tfw/components/fsm/fsm_updater.py @@ -5,7 +5,7 @@ class FSMUpdater: @property def fsm_update(self): return { - 'key': 'fsm.announce', + 'key': 'fsm.update', **self.fsm_update_data } diff --git a/tfw/components/process_management/log_inotify_observer.py b/tfw/components/process_management/log_inotify_observer.py index 4ea88cf..97b6767 100644 --- a/tfw/components/process_management/log_inotify_observer.py +++ b/tfw/components/process_management/log_inotify_observer.py @@ -36,7 +36,7 @@ class LogInotifyObserver(InotifyObserver, ProcessLogManager): def on_modified(self, event): self._connector.send_message({ - 'key': 'log.new', + 'key': 'process.log.new', 'stdout': self.read_stdout(self.process_name, tail=self.log_tail), 'stderr': self.read_stderr(self.process_name, tail=self.log_tail) }, Scope.BROADCAST) diff --git a/tfw/components/process_management/process_log_handler.py b/tfw/components/process_management/process_log_handler.py index b23874e..144b1be 100644 --- a/tfw/components/process_management/process_log_handler.py +++ b/tfw/components/process_management/process_log_handler.py @@ -6,7 +6,7 @@ LOG = logging.getLogger(__name__) class ProcessLogHandler: - keys = ['log.set'] + keys = ['process.log.set'] def __init__(self, *, process_name, supervisor_uri, log_tail=0): self.connector, self._monitor = None, None @@ -15,7 +15,7 @@ class ProcessLogHandler: self._initial_log_tail = log_tail self.command_handlers = { - 'log.set': self.handle_set + 'process.log.set': self.handle_set } def start(self): diff --git a/tfw/internals/event_handling/fsm_aware.py b/tfw/internals/event_handling/fsm_aware.py index b700dba..b888220 100644 --- a/tfw/internals/event_handling/fsm_aware.py +++ b/tfw/internals/event_handling/fsm_aware.py @@ -6,11 +6,7 @@ LOG = logging.getLogger(__name__) class FSMAware: - keys = ['fsm.announce'] - """ - Base class for stuff that has to be aware of the framework FSM. - This is done by processing 'fsm_update' messages. - """ + keys = ['fsm.update'] def __init__(self): self.fsm_state = None self.fsm_in_accepted_state = False @@ -18,7 +14,7 @@ class FSMAware: self._auth_key = KeyManager().auth_key def process_message(self, message): - if message['key'] == 'fsm.announce': + if message['key'] == 'fsm.update': if verify_message(self._auth_key, message): self._handle_fsm_update(message) From b5e53cb9467acc2528e3d326db6a635349256cf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 8 Aug 2019 15:05:37 +0200 Subject: [PATCH 114/174] Use prefix matching in ZMQConnector --- tfw/components/frontend/frontend_handler.py | 40 ++++++++++--------- tfw/components/frontend/message_storage.py | 29 +++++++++----- tfw/components/fsm/fsm_handler.py | 3 +- tfw/components/ide/ide_handler.py | 2 +- .../process_management/process_handler.py | 6 ++- .../process_management/process_log_handler.py | 2 +- tfw/components/snapshots/snapshot_handler.py | 27 +++++-------- tfw/components/terminal/terminal_handler.py | 2 +- tfw/internals/networking/zmq_connector.py | 16 +------- 9 files changed, 62 insertions(+), 65 deletions(-) diff --git a/tfw/components/frontend/frontend_handler.py b/tfw/components/frontend/frontend_handler.py index 7e35bc5..3b8a76a 100644 --- a/tfw/components/frontend/frontend_handler.py +++ b/tfw/components/frontend/frontend_handler.py @@ -4,26 +4,25 @@ from .message_storage import FrontendMessageStorage class FrontendHandler: - # TODO: do not store dashboard messages like reloadFrontend - keys = [ - 'console.read', - 'console.write', - 'console.showLiveLogs', - 'console.rewriteContentWithProcessLogsOnDeploy', - 'dashboard.layout', - 'dashboard.hideMessages', - 'dashboard.terminalMenuItem', - 'dashboard.reloadFrontend', - 'dashboard.reloadIframe', - 'message.config', - 'message.queue', - 'message.send' - ] + # keys = [ + # 'console.read', + # 'console.write', + # 'console.showLiveLogs', + # 'console.rewriteContentWithProcessLogsOnDeploy', + # 'dashboard.layout', + # 'dashboard.hideMessages', + # 'dashboard.terminalMenuItem', + # 'dashboard.reloadFrontend', + # 'dashboard.reloadIframe', + # 'message.config', + # 'message.queue', + # 'message.send' + # ] + keys = ['console', 'dashboard', 'message', 'ide.read', 'recover'] def __init__(self): self.connector = None - self.keys = [*type(self).keys, 'recover'] - self._frontend_message_storage = FrontendMessageStorage(type(self).keys) + self._frontend_message_storage = FrontendMessageStorage() def send_message(self, message): self.connector.send_message(message, scope=Scope.WEBSOCKET) @@ -32,7 +31,12 @@ class FrontendHandler: self._frontend_message_storage.save_message(message) if message['key'] == 'recover': self.recover_frontend() - self.send_message(message) + if self._filter_message(message): + self.send_message(message) + + @staticmethod + def _filter_message(message): + return not message['key'].startswith('ide') def recover_frontend(self): for message in self._frontend_message_storage.messages: diff --git a/tfw/components/frontend/message_storage.py b/tfw/components/frontend/message_storage.py index 1c63ffc..2478827 100644 --- a/tfw/components/frontend/message_storage.py +++ b/tfw/components/frontend/message_storage.py @@ -29,16 +29,25 @@ class MessageStorage(ABC): class FrontendMessageStorage(MessageStorage): - def __init__(self, keys): - self._keys = keys - super().__init__() - def _filter_message(self, message): - key = message['key'] - return key in self._keys + return message['key'].startswith(( + 'console.write', + 'dashboard.layout', + 'dashboard.terminalMenuItem', + 'message.send', + 'message.config', + 'ide.read' + )) def _transform_message(self, message): - if message['key'] == 'queueMessages': - yield from MessageSender.generate_messages_from_queue(message) - else: - yield message + transformations = { + 'message.queue': MessageSender.generate_messages_from_queue, + 'ide.read': self._delete_ide_content + } + if message['key'] in transformations: + yield from transformations[message['key']](message) + + @staticmethod + def _delete_ide_content(message): + del message['content'] + yield message diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index 5d28c1f..24b8231 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -9,7 +9,8 @@ LOG = logging.getLogger(__name__) class FSMHandler: - keys = ['fsm.step', 'fsm.announce'] + keys = ['fsm'] + def __init__(self, *, fsm_type): self.fsm = fsm_type() self._fsm_updater = FSMUpdater(self.fsm) diff --git a/tfw/components/ide/ide_handler.py b/tfw/components/ide/ide_handler.py index f7435d6..bf1307b 100644 --- a/tfw/components/ide/ide_handler.py +++ b/tfw/components/ide/ide_handler.py @@ -33,7 +33,7 @@ BUILD_ARTIFACTS = ( class IdeHandler: - keys = ['ide.read', 'ide.write'] + keys = ['ide'] def __init__(self, *, patterns, initial_file=''): self.connector = None diff --git a/tfw/components/process_management/process_handler.py b/tfw/components/process_management/process_handler.py index 80e1a4e..33e58fa 100644 --- a/tfw/components/process_management/process_handler.py +++ b/tfw/components/process_management/process_handler.py @@ -9,7 +9,8 @@ LOG = logging.getLogger(__name__) class ProcessHandler(ProcessManager, ProcessLogManager): - keys = ['process.start', 'process.stop', 'process.restart'] + keys = ['process'] + def __init__(self, *, supervisor_uri, log_tail=0): ProcessManager.__init__(self, supervisor_uri) ProcessLogManager.__init__(self, supervisor_uri) @@ -38,4 +39,5 @@ class ProcessHandler(ProcessManager, ProcessLogManager): ) connector.send_message(message, scope=Scope.WEBSOCKET) except KeyError: - LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) + if not message['key'].startswith('process.log'): + LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/tfw/components/process_management/process_log_handler.py b/tfw/components/process_management/process_log_handler.py index 144b1be..9e3bb42 100644 --- a/tfw/components/process_management/process_log_handler.py +++ b/tfw/components/process_management/process_log_handler.py @@ -6,7 +6,7 @@ LOG = logging.getLogger(__name__) class ProcessLogHandler: - keys = ['process.log.set'] + keys = ['process.log'] def __init__(self, *, process_name, supervisor_uri, log_tail=0): self.connector, self._monitor = None, None diff --git a/tfw/components/snapshots/snapshot_handler.py b/tfw/components/snapshots/snapshot_handler.py index f992b07..0c8f797 100644 --- a/tfw/components/snapshots/snapshot_handler.py +++ b/tfw/components/snapshots/snapshot_handler.py @@ -6,8 +6,6 @@ from datetime import datetime from dateutil import parser as dateparser -from tfw.internals.networking import Scope - from .snapshot_provider import SnapshotProvider LOG = logging.getLogger(__name__) @@ -23,9 +21,9 @@ class SnapshotHandler: self.init_snapshot_providers(directories) self.command_handlers = { - 'take_snapshot': self.handle_take_snapshot, - 'restore_snapshot': self.handle_restore_snapshot, - 'exclude': self.handle_exclude + 'snapshot.take': self.handle_take_snapshot, + 'snapshot.restore': self.handle_restore_snapshot, + 'snapshot.exclude': self.handle_exclude } def init_snapshot_providers(self, directories): @@ -45,23 +43,20 @@ class SnapshotHandler: makedirs(git_dir, exist_ok=True) return git_dir - def handle_event(self, message, connector): + def handle_event(self, message, _): try: - data = message['data'] - message['data'] = self.command_handlers[data['command']](data) - connector.send_message(message, scope=Scope.WEBSOCKET) + self.command_handlers[message['key']](message) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) - def handle_take_snapshot(self, data): + def handle_take_snapshot(self, _): LOG.debug('Taking snapshots of directories %s', self.snapshot_providers.keys()) for provider in self.snapshot_providers.values(): provider.take_snapshot() - return data - def handle_restore_snapshot(self, data): + def handle_restore_snapshot(self, message): date = dateparser.parse( - data.get( + message.get( 'value', datetime.now().isoformat() ) @@ -73,13 +68,11 @@ class SnapshotHandler: ) for provider in self.snapshot_providers.values(): provider.restore_snapshot(date) - return data - def handle_exclude(self, data): - exclude_unix_patterns = data['value'] + def handle_exclude(self, message): + exclude_unix_patterns = message['value'] if not isinstance(exclude_unix_patterns, list): raise KeyError for provider in self.snapshot_providers.values(): provider.exclude = exclude_unix_patterns - return data diff --git a/tfw/components/terminal/terminal_handler.py b/tfw/components/terminal/terminal_handler.py index 7d3e041..dd17e97 100644 --- a/tfw/components/terminal/terminal_handler.py +++ b/tfw/components/terminal/terminal_handler.py @@ -7,7 +7,7 @@ LOG = logging.getLogger(__name__) class TerminalHandler: - keys = ['terminal.write'] + keys = ['terminal'] def __init__(self, *, port, user, working_directory, histfile): self.connector, self._historymonitor = None, None diff --git a/tfw/internals/networking/zmq_connector.py b/tfw/internals/networking/zmq_connector.py index 7dfa77c..99920c7 100644 --- a/tfw/internals/networking/zmq_connector.py +++ b/tfw/internals/networking/zmq_connector.py @@ -15,8 +15,6 @@ LOG = logging.getLogger(__name__) class ZMQDownlinkConnector: def __init__(self, connect_addr): - self.keys = [] - self._on_recv_callback = None self._zmq_sub_socket = zmq.Context.instance().socket(zmq.SUB) self._zmq_sub_socket.setsockopt(zmq.RCVHWM, 0) self._zmq_sub_socket.connect(connect_addr) @@ -25,19 +23,14 @@ class ZMQDownlinkConnector: def subscribe(self, *keys): for key in keys: self._zmq_sub_socket.setsockopt_string(zmq.SUBSCRIBE, key) - self.keys.append(key) def unsubscribe(self, *keys): for key in keys: self._zmq_sub_socket.setsockopt_string(zmq.UNSUBSCRIBE, key) - self.keys.remove(key) def register_callback(self, callback): - if callback: - self._on_recv_callback = callback - self._zmq_sub_stream.on_recv(with_deserialize_tfw_msg(self._on_recv)) - else: - self._zmq_sub_stream.on_recv(None) + callback = with_deserialize_tfw_msg(callback) if callback else None + self._zmq_sub_stream.on_recv(callback) def recv_message(self, *, block=True): if self._zmq_sub_stream.receiving(): @@ -48,11 +41,6 @@ class ZMQDownlinkConnector: except zmq.ZMQError: raise IOError("No data available to recv!") - def _on_recv(self, message): - key = message['key'] - if key in self.keys or '' in self.keys: - self._on_recv_callback(message) - def close(self): self._zmq_sub_stream.close() From 7ba502277fd77cfeae1f7b4d6ec250e3b9b012d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 9 Aug 2019 10:56:38 +0200 Subject: [PATCH 115/174] Remove accidental comment --- tfw/components/frontend/frontend_handler.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tfw/components/frontend/frontend_handler.py b/tfw/components/frontend/frontend_handler.py index 3b8a76a..dfb50bf 100644 --- a/tfw/components/frontend/frontend_handler.py +++ b/tfw/components/frontend/frontend_handler.py @@ -4,20 +4,6 @@ from .message_storage import FrontendMessageStorage class FrontendHandler: - # keys = [ - # 'console.read', - # 'console.write', - # 'console.showLiveLogs', - # 'console.rewriteContentWithProcessLogsOnDeploy', - # 'dashboard.layout', - # 'dashboard.hideMessages', - # 'dashboard.terminalMenuItem', - # 'dashboard.reloadFrontend', - # 'dashboard.reloadIframe', - # 'message.config', - # 'message.queue', - # 'message.send' - # ] keys = ['console', 'dashboard', 'message', 'ide.read', 'recover'] def __init__(self): From 0b7557f1989f55b324f137436b61c08ba7877e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 9 Aug 2019 10:59:27 +0200 Subject: [PATCH 116/174] Fix frontend message replay broken during refactor --- tfw/components/frontend/message_storage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tfw/components/frontend/message_storage.py b/tfw/components/frontend/message_storage.py index 2478827..16901e8 100644 --- a/tfw/components/frontend/message_storage.py +++ b/tfw/components/frontend/message_storage.py @@ -46,6 +46,8 @@ class FrontendMessageStorage(MessageStorage): } if message['key'] in transformations: yield from transformations[message['key']](message) + else: + yield message @staticmethod def _delete_ide_content(message): From 0549c5e4c84018996792ea2e49df58747b200d15 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 12 Aug 2019 11:57:54 +0200 Subject: [PATCH 117/174] Fix message filtering issue --- tfw/components/frontend/message_storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tfw/components/frontend/message_storage.py b/tfw/components/frontend/message_storage.py index 16901e8..78bba26 100644 --- a/tfw/components/frontend/message_storage.py +++ b/tfw/components/frontend/message_storage.py @@ -34,8 +34,9 @@ class FrontendMessageStorage(MessageStorage): 'console.write', 'dashboard.layout', 'dashboard.terminalMenuItem', - 'message.send', 'message.config', + 'message.send', + 'message.queue', 'ide.read' )) From 9e49a93df34edea94753c2291e9d7eb4e24ed2bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 12 Aug 2019 16:25:03 +0200 Subject: [PATCH 118/174] Fix event handler pytest.raises test cases --- .../event_handling/test_event_handler.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tfw/internals/event_handling/test_event_handler.py b/tfw/internals/event_handling/test_event_handler.py index 03f4141..6398627 100644 --- a/tfw/internals/event_handling/test_event_handler.py +++ b/tfw/internals/event_handling/test_event_handler.py @@ -89,9 +89,9 @@ def test_build_from_object(test_keys, test_msg): assert mock_eh.connector is eh.connector with pytest.raises(RuntimeError) as err: eh.connector.simulate_message(test_msg) - msg, keys = err.args - assert msg == test_msg - assert keys == test_keys + msg, keys = err.value.args + assert msg == test_msg + assert keys == test_keys assert not mock_eh.cleaned_up eh.stop() assert mock_eh.cleaned_up @@ -119,16 +119,15 @@ def test_build_from_simple_object(test_keys, test_msg): class SimpleMockEventHandler: # pylint: disable=no-self-use def handle_event(self, message, connector): - raise RuntimeError(message, connector) + raise RuntimeError(message) mock_eh = SimpleMockEventHandler() eh = MockEventHandlerFactory().build(mock_eh, keys=test_keys) with pytest.raises(RuntimeError) as err: eh.connector.simulate_message(test_msg) - msg, keys = err.args - assert msg == test_msg - assert keys == test_keys + msg = err.value.args[0] + assert msg == test_msg def test_build_from_callable(test_keys, test_msg): @@ -156,9 +155,9 @@ def test_build_from_function(test_keys, test_msg): assert eh.connector.keys == test_keys with pytest.raises(RuntimeError) as err: eh.connector.simulate_message(test_msg) - msg, keys = err.args - assert msg == test_msg - assert keys == test_keys + msg, keys = err.value.args + assert msg == test_msg + assert keys == test_keys def test_build_from_lambda(test_keys, test_msg): From 8f7715565ed8c7e7ac4a77d5488ceae26a4801c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Aug 2019 13:49:25 +0200 Subject: [PATCH 119/174] Silence watchdog's internal deprecation warning when using pytest --- pytest.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 72a218a..70c3cf1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,4 @@ [pytest] -filterwarnings = ignore::DeprecationWarning:zmq +filterwarnings = + ignore::DeprecationWarning:zmq + ignore::DeprecationWarning:watchdog From 0df378fb9228f7d1ca235b43567bfcdacbdfd4a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Aug 2019 13:49:58 +0200 Subject: [PATCH 120/174] Fix FileManager unit test cases for macOS (symlinked TMPDIR) --- tfw/components/ide/file_manager/test_file_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tfw/components/ide/file_manager/test_file_manager.py b/tfw/components/ide/file_manager/test_file_manager.py index 08e4f25..d321ea3 100644 --- a/tfw/components/ide/file_manager/test_file_manager.py +++ b/tfw/components/ide/file_manager/test_file_manager.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from secrets import token_urlsafe from os import mkdir, symlink -from os.path import join +from os.path import join, realpath from pathlib import Path from tempfile import TemporaryDirectory @@ -42,6 +42,7 @@ def generate_name(): @pytest.fixture() def context(): with TemporaryDirectory() as workdir: + workdir = realpath(workdir) # macOS uses a symlinked TMPDIR subdir = join(workdir, generate_name()) subfile = join(subdir, generate_name() + '.txt') mkdir(subdir) From f5582f0207f691535ca99dc75f007cd2dda0837c Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 14 Aug 2019 14:12:31 +0200 Subject: [PATCH 121/174] Implement inotify based pipe connector --- tfw/components/pipe_connector/__init__.py | 1 + .../pipe_connector/pipe_connector.py | 69 +++++++++ .../proxy_pipe_connector_handler.py | 43 ++++++ .../pipe_connector/test_pipe_connector.py | 144 ++++++++++++++++++ .../log_inotify_observer.py | 6 - tfw/internals/inotify/inotify.py | 4 +- 6 files changed, 260 insertions(+), 7 deletions(-) create mode 100644 tfw/components/pipe_connector/__init__.py create mode 100644 tfw/components/pipe_connector/pipe_connector.py create mode 100644 tfw/components/pipe_connector/proxy_pipe_connector_handler.py create mode 100644 tfw/components/pipe_connector/test_pipe_connector.py diff --git a/tfw/components/pipe_connector/__init__.py b/tfw/components/pipe_connector/__init__.py new file mode 100644 index 0000000..07f335a --- /dev/null +++ b/tfw/components/pipe_connector/__init__.py @@ -0,0 +1 @@ +from .proxy_pipe_connector_handler import ProxyPipeConnectorHandler diff --git a/tfw/components/pipe_connector/pipe_connector.py b/tfw/components/pipe_connector/pipe_connector.py new file mode 100644 index 0000000..d1500c2 --- /dev/null +++ b/tfw/components/pipe_connector/pipe_connector.py @@ -0,0 +1,69 @@ +from stat import S_ISFIFO +from os import access, mkdir, stat, R_OK +from os.path import exists + +from pipe_io_server import PipeReaderServer, PipeWriterServer + +from tfw.internals.inotify import InotifyObserver, InotifyFileCreatedEvent, InotifyFileDeletedEvent + + +class PipeConnector: + def __init__(self, path): + self.recv_pipes, self.send_pipes = {}, {} + self.observer = self.build_observer(path) + self.observer.on_any_event = self._on_any_event + self.observer.start() + + def build_observer(self, path): # pylint: disable=no-self-use + if not exists(path): + mkdir(path) + if not access(path, R_OK): + raise ValueError('Path does not exist or is not accessible.') + observer = InotifyObserver(path, patterns=['*send*', '*recv*']) + return observer + + def _on_any_event(self, event): + path = event.src_path + if isinstance(event, InotifyFileCreatedEvent) and self._is_pipe(path): + self._create_pipe(path) + elif isinstance(event, InotifyFileDeletedEvent): + self._delete_pipe(path) + + @staticmethod + def _is_pipe(path): + return exists(path) and S_ISFIFO(stat(path).st_mode) + + def _create_pipe(self, path): + if self._find_pipe(path): + return + server = None + if 'recv' in path: + pipes, server = self.recv_pipes, self.build_writer(path) + elif 'send' in path: + pipes, server = self.send_pipes, self.build_reader(path) + if server: + server.start() + pipes[path] = server + + def _find_pipe(self, path): + if path in self.recv_pipes.keys(): + return self.recv_pipes + if path in self.send_pipes.keys(): + return self.send_pipes + return None + + def build_reader(self, path): # pylint: disable=no-self-use + return PipeReaderServer(path, manager_pipes=False) + + def build_writer(self, path): # pylint: disable=no-self-use + return PipeWriterServer(path, manage_pipes=False) + + def _delete_pipe(self, path): + pipes = self._find_pipe(path) + if pipes: + pipes[path].stop() + del pipes[path] + + def broadcast(self, message): + for _, server in self.recv_pipes.items(): + server.send_message(message) diff --git a/tfw/components/pipe_connector/proxy_pipe_connector_handler.py b/tfw/components/pipe_connector/proxy_pipe_connector_handler.py new file mode 100644 index 0000000..b3b20f7 --- /dev/null +++ b/tfw/components/pipe_connector/proxy_pipe_connector_handler.py @@ -0,0 +1,43 @@ +import logging +from threading import Lock +from json import dumps, loads, JSONDecodeError + +from pipe_io_server import PipeReaderServer + +from .pipe_connector import PipeConnector + +LOG = logging.getLogger(__name__) + + +class ProxyPipeConnectorHandler: + keys = [''] + + def __init__(self, path): + self.connector, self.pipes = None, None + self.path = path + + def start(self): + self.pipes = ProxyPipeConnector(self.path, self.connector) + + def handle_event(self, message, _): + self.pipes.broadcast(dumps(message).encode()) + + +class ProxyPipeConnector(PipeConnector): + def __init__(self, path, connector): + self.connector = connector + self.mutex = Lock() + super().__init__(path) + + def build_reader(self, path): + reader = PipeReaderServer(path, manage_pipes=False) + reader.handle_message = self._handle_message + return reader + + def _handle_message(self, message): + try: + json_object = loads(message) + with self.mutex: + self.connector.send_message(json_object) + except JSONDecodeError: + LOG.error('Received invalid JSON message: %s', message) diff --git a/tfw/components/pipe_connector/test_pipe_connector.py b/tfw/components/pipe_connector/test_pipe_connector.py new file mode 100644 index 0000000..bdf8a04 --- /dev/null +++ b/tfw/components/pipe_connector/test_pipe_connector.py @@ -0,0 +1,144 @@ +# pylint: disable=redefined-outer-name +from enum import Enum +from dataclasses import dataclass +from json import dumps +from secrets import token_urlsafe +from os import urandom, mkfifo, mkdir, mknod +from os.path import join +from tempfile import TemporaryDirectory + +from tfw.internals.inotify import InotifyFileCreatedEvent, InotifyFileDeletedEvent + +import pytest + +from .proxy_pipe_connector_handler import ProxyPipeConnector + + +class Action(Enum): + SEND = 'send' + RECV = 'recv' + + +@dataclass +class PipeContext: + workdir: str + pipes: ProxyPipeConnector + + def emit_pipe_creation_event(self, action, inode_creator): + filename = self.join(f'{self.generate_name()}_{action.value}') + inode_creator(filename) + self.pipes.observer.on_any_event(InotifyFileCreatedEvent(filename)) + return filename + + def join(self, path): + return join(self.workdir, path) + + @staticmethod + def generate_name(): + return urandom(4).hex() + + +class MockPipeConnector(ProxyPipeConnector): + def __init__(self, path, connector): + self.reader_events, self.writer_events = [], [] + super().__init__(path, connector) + + def build_observer(self, path): + return MockObserver() + + def build_reader(self, path): + self.reader_events.append(path) + reader = MockPipeServer() + reader.handle_message = self._handle_message + return reader + + def build_writer(self, path): + self.writer_events.append(path) + return MockPipeServer() + + +class MockObserver: + def start(self): + pass + + def on_any_event(self, event): + pass + + +class MockPipeServer: + def __init__(self): + self.messages = [] + + def handle_message(self, message): + pass + + def send_message(self, message): + self.messages.append(message) + + def start(self): + pass + + def stop(self): + pass + + +class MockConnector: # pylint: disable=too-few-public-methods + def __init__(self): + self.messages = [] + + def send_message(self, message): + self.messages.append(message) + + +@pytest.fixture +def context(): + with TemporaryDirectory() as workdir: + yield PipeContext(workdir, MockPipeConnector(workdir, MockConnector())) + + +def test_pipe_creation_deletion(context): + cases = [ + (Action.RECV, context.pipes.recv_pipes, context.pipes.writer_events), + (Action.SEND, context.pipes.send_pipes, context.pipes.reader_events) + ] + + for action, pipes, events in cases: + path = context.emit_pipe_creation_event(action, mkfifo) + assert events[-1] == path + assert path in pipes.keys() + context.pipes.observer.on_any_event(InotifyFileDeletedEvent(path)) + assert path not in pipes.keys() + + +def test_handle_message(context): + path = context.emit_pipe_creation_event(Action.SEND, mkfifo) + payload = {'key': token_urlsafe(16)} + context.pipes.send_pipes[path].handle_message(dumps(payload)) + assert context.pipes.connector.messages[-1] == payload + context.pipes.send_pipes[path].handle_message(token_urlsafe(32)) + assert len(context.pipes.connector.messages) == 1 + + +def test_broadcast(context): + paths = [ + context.emit_pipe_creation_event(Action.RECV, mkfifo) + for _ in range(32) + ] + payload = {'key': token_urlsafe(16)} + + context.pipes.broadcast(payload) + for path in paths: + assert context.pipes.recv_pipes[path].messages[-1] == payload + + +def test_inode_types(context): + cases = [ + (Action.RECV, context.pipes.recv_pipes, mkdir), + (Action.SEND, context.pipes.send_pipes, mkdir), + (Action.RECV, context.pipes.recv_pipes, mknod), + (Action.SEND, context.pipes.send_pipes, mknod) + ] + + for action, pipes, creator in cases: + path = context.emit_pipe_creation_event(action, creator) + assert path not in pipes.keys() diff --git a/tfw/components/process_management/log_inotify_observer.py b/tfw/components/process_management/log_inotify_observer.py index 97b6767..1c80dd1 100644 --- a/tfw/components/process_management/log_inotify_observer.py +++ b/tfw/components/process_management/log_inotify_observer.py @@ -8,7 +8,6 @@ from .supervisor import ProcessLogManager class LogInotifyObserver(InotifyObserver, ProcessLogManager): def __init__(self, connector, process_name, supervisor_uri, log_tail=0): - self._prevent_log_recursion() self._connector = connector self._process_name = process_name self.log_tail = log_tail @@ -16,11 +15,6 @@ class LogInotifyObserver(InotifyObserver, ProcessLogManager): ProcessLogManager.__init__(self, supervisor_uri) InotifyObserver.__init__(self, self._get_logfiles()) - @staticmethod - def _prevent_log_recursion(): - # This is done to prevent inotify event logs triggering themselves (infinite log recursion) - logging.getLogger('watchdog.observers.inotify_buffer').propagate = False - def _get_logfiles(self): self._procinfo = self.supervisor.getProcessInfo(self._process_name) return self._procinfo['stdout_logfile'], self._procinfo['stderr_logfile'] diff --git a/tfw/internals/inotify/inotify.py b/tfw/internals/inotify/inotify.py index 4f69a02..2749ca6 100644 --- a/tfw/internals/inotify/inotify.py +++ b/tfw/internals/inotify/inotify.py @@ -1,5 +1,5 @@ # pylint: disable=too-few-public-methods - +import logging from typing import Iterable from time import time from os.path import abspath, dirname, isdir @@ -11,6 +11,8 @@ from watchdog.events import ( DirCreatedEvent, DirModifiedEvent, DirMovedEvent, DirDeletedEvent ) +logging.getLogger('watchdog.observers.inotify_buffer').propagate = False + class InotifyEvent: def __init__(self, src_path): From dc0615c11e1a4f7da98fe558f913fcb6a611463a Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 14 Aug 2019 15:42:11 +0200 Subject: [PATCH 122/174] Fix issues in PR #63 --- tfw/components/pipe_io/__init__.py | 1 + .../{ => pipe_io}/pipe_connector/__init__.py | 0 .../pipe_connector/pipe_connector.py | 29 +++++---- .../proxy_pipe_connector_handler.py | 0 .../test_proxy_pipe_connector.py} | 62 ++++++++++++------- tfw/internals/inotify/inotify.py | 1 + 6 files changed, 57 insertions(+), 36 deletions(-) rename tfw/components/{ => pipe_io}/pipe_connector/__init__.py (100%) rename tfw/components/{ => pipe_io}/pipe_connector/pipe_connector.py (72%) rename tfw/components/{ => pipe_io}/pipe_connector/proxy_pipe_connector_handler.py (100%) rename tfw/components/{pipe_connector/test_pipe_connector.py => pipe_io/pipe_connector/test_proxy_pipe_connector.py} (59%) diff --git a/tfw/components/pipe_io/__init__.py b/tfw/components/pipe_io/__init__.py index 43a5b98..54bc368 100644 --- a/tfw/components/pipe_io/__init__.py +++ b/tfw/components/pipe_io/__init__.py @@ -1 +1,2 @@ from .pipe_io_handler import PipeIOHandler, PipeIOHandlerBase, TransformerPipeIOHandler, CommandHandler +from .pipe_connector import ProxyPipeConnectorHandler diff --git a/tfw/components/pipe_connector/__init__.py b/tfw/components/pipe_io/pipe_connector/__init__.py similarity index 100% rename from tfw/components/pipe_connector/__init__.py rename to tfw/components/pipe_io/pipe_connector/__init__.py diff --git a/tfw/components/pipe_connector/pipe_connector.py b/tfw/components/pipe_io/pipe_connector/pipe_connector.py similarity index 72% rename from tfw/components/pipe_connector/pipe_connector.py rename to tfw/components/pipe_io/pipe_connector/pipe_connector.py index d1500c2..1f47ace 100644 --- a/tfw/components/pipe_connector/pipe_connector.py +++ b/tfw/components/pipe_io/pipe_connector/pipe_connector.py @@ -8,6 +8,8 @@ from tfw.internals.inotify import InotifyObserver, InotifyFileCreatedEvent, Inot class PipeConnector: + reader_pattern, writer_pattern = 'send', 'recv' + def __init__(self, path): self.recv_pipes, self.send_pipes = {}, {} self.observer = self.build_observer(path) @@ -19,15 +21,15 @@ class PipeConnector: mkdir(path) if not access(path, R_OK): raise ValueError('Path does not exist or is not accessible.') - observer = InotifyObserver(path, patterns=['*send*', '*recv*']) - return observer + return InotifyObserver(path, patterns=[f'*{self.reader_pattern}*', f'*{self.writer_pattern}*']) def _on_any_event(self, event): path = event.src_path - if isinstance(event, InotifyFileCreatedEvent) and self._is_pipe(path): - self._create_pipe(path) - elif isinstance(event, InotifyFileDeletedEvent): - self._delete_pipe(path) + if self._is_pipe(path): + if isinstance(event, InotifyFileCreatedEvent): + self._create_pipe(path) + elif isinstance(event, InotifyFileDeletedEvent): + self._delete_pipe(path) @staticmethod def _is_pipe(path): @@ -37,23 +39,24 @@ class PipeConnector: if self._find_pipe(path): return server = None - if 'recv' in path: + if self.writer_pattern in path: pipes, server = self.recv_pipes, self.build_writer(path) - elif 'send' in path: + elif self.reader_pattern in path: pipes, server = self.send_pipes, self.build_reader(path) if server: server.start() pipes[path] = server def _find_pipe(self, path): + pipes = None if path in self.recv_pipes.keys(): - return self.recv_pipes + pipes = self.recv_pipes if path in self.send_pipes.keys(): - return self.send_pipes - return None + pipes = self.send_pipes + return pipes def build_reader(self, path): # pylint: disable=no-self-use - return PipeReaderServer(path, manager_pipes=False) + raise NotImplementedError() def build_writer(self, path): # pylint: disable=no-self-use return PipeWriterServer(path, manage_pipes=False) @@ -65,5 +68,5 @@ class PipeConnector: del pipes[path] def broadcast(self, message): - for _, server in self.recv_pipes.items(): + for server in self.recv_pipes.values(): server.send_message(message) diff --git a/tfw/components/pipe_connector/proxy_pipe_connector_handler.py b/tfw/components/pipe_io/pipe_connector/proxy_pipe_connector_handler.py similarity index 100% rename from tfw/components/pipe_connector/proxy_pipe_connector_handler.py rename to tfw/components/pipe_io/pipe_connector/proxy_pipe_connector_handler.py diff --git a/tfw/components/pipe_connector/test_pipe_connector.py b/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py similarity index 59% rename from tfw/components/pipe_connector/test_pipe_connector.py rename to tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py index bdf8a04..12a541a 100644 --- a/tfw/components/pipe_connector/test_pipe_connector.py +++ b/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py @@ -11,6 +11,7 @@ from tfw.internals.inotify import InotifyFileCreatedEvent, InotifyFileDeletedEve import pytest +from .pipe_connector import PipeConnector from .proxy_pipe_connector_handler import ProxyPipeConnector @@ -91,54 +92,69 @@ class MockConnector: # pylint: disable=too-few-public-methods @pytest.fixture -def context(): +def workdir(): with TemporaryDirectory() as workdir: - yield PipeContext(workdir, MockPipeConnector(workdir, MockConnector())) + yield workdir -def test_pipe_creation_deletion(context): +@pytest.fixture +def context(workdir): + yield PipeContext(workdir, PipeConnector(workdir)) + + +@pytest.fixture +def mock_context(workdir): + yield PipeContext(workdir, MockPipeConnector(workdir, MockConnector())) + + +def test_pipe_creation_deletion(mock_context): cases = [ - (Action.RECV, context.pipes.recv_pipes, context.pipes.writer_events), - (Action.SEND, context.pipes.send_pipes, context.pipes.reader_events) + (Action.RECV, mock_context.pipes.recv_pipes, mock_context.pipes.writer_events), + (Action.SEND, mock_context.pipes.send_pipes, mock_context.pipes.reader_events) ] for action, pipes, events in cases: - path = context.emit_pipe_creation_event(action, mkfifo) + path = mock_context.emit_pipe_creation_event(action, mkfifo) assert events[-1] == path assert path in pipes.keys() - context.pipes.observer.on_any_event(InotifyFileDeletedEvent(path)) + mock_context.pipes.observer.on_any_event(InotifyFileDeletedEvent(path)) assert path not in pipes.keys() -def test_handle_message(context): - path = context.emit_pipe_creation_event(Action.SEND, mkfifo) +def test_handle_message(mock_context): + path = mock_context.emit_pipe_creation_event(Action.SEND, mkfifo) payload = {'key': token_urlsafe(16)} - context.pipes.send_pipes[path].handle_message(dumps(payload)) - assert context.pipes.connector.messages[-1] == payload - context.pipes.send_pipes[path].handle_message(token_urlsafe(32)) - assert len(context.pipes.connector.messages) == 1 + mock_context.pipes.send_pipes[path].handle_message(dumps(payload)) + assert mock_context.pipes.connector.messages[-1] == payload + mock_context.pipes.send_pipes[path].handle_message(token_urlsafe(32)) + assert len(mock_context.pipes.connector.messages) == 1 -def test_broadcast(context): +def test_broadcast(mock_context): paths = [ - context.emit_pipe_creation_event(Action.RECV, mkfifo) + mock_context.emit_pipe_creation_event(Action.RECV, mkfifo) for _ in range(32) ] payload = {'key': token_urlsafe(16)} - context.pipes.broadcast(payload) + mock_context.pipes.broadcast(payload) for path in paths: - assert context.pipes.recv_pipes[path].messages[-1] == payload + assert mock_context.pipes.recv_pipes[path].messages[-1] == payload -def test_inode_types(context): +def test_inode_types(mock_context): cases = [ - (Action.RECV, context.pipes.recv_pipes, mkdir), - (Action.SEND, context.pipes.send_pipes, mkdir), - (Action.RECV, context.pipes.recv_pipes, mknod), - (Action.SEND, context.pipes.send_pipes, mknod) + (Action.RECV, mock_context.pipes.recv_pipes, mkdir), + (Action.SEND, mock_context.pipes.send_pipes, mkdir), + (Action.RECV, mock_context.pipes.recv_pipes, mknod), + (Action.SEND, mock_context.pipes.send_pipes, mknod) ] for action, pipes, creator in cases: - path = context.emit_pipe_creation_event(action, creator) + path = mock_context.emit_pipe_creation_event(action, creator) assert path not in pipes.keys() + + +def test_build_reader_implemented(context): + with pytest.raises(NotImplementedError): + context.emit_pipe_creation_event(Action.SEND, mkfifo) diff --git a/tfw/internals/inotify/inotify.py b/tfw/internals/inotify/inotify.py index 2749ca6..867a704 100644 --- a/tfw/internals/inotify/inotify.py +++ b/tfw/internals/inotify/inotify.py @@ -11,6 +11,7 @@ from watchdog.events import ( DirCreatedEvent, DirModifiedEvent, DirMovedEvent, DirDeletedEvent ) +# This is done to prevent inotify event logs triggering themselves (infinite log recursion) logging.getLogger('watchdog.observers.inotify_buffer').propagate = False From 2fb971d317e9a24c8f5a8fa81de40a0e5704fb0b Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 14 Aug 2019 16:05:55 +0200 Subject: [PATCH 123/174] Fix inode creation issue on MacOS --- .../pipe_io/pipe_connector/test_proxy_pipe_connector.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py b/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py index 12a541a..3961b28 100644 --- a/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py +++ b/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py @@ -3,8 +3,9 @@ from enum import Enum from dataclasses import dataclass from json import dumps from secrets import token_urlsafe -from os import urandom, mkfifo, mkdir, mknod +from os import urandom, mkfifo, mkdir from os.path import join +from pathlib import Path from tempfile import TemporaryDirectory from tfw.internals.inotify import InotifyFileCreatedEvent, InotifyFileDeletedEvent @@ -141,13 +142,13 @@ def test_broadcast(mock_context): for path in paths: assert mock_context.pipes.recv_pipes[path].messages[-1] == payload - def test_inode_types(mock_context): + touch = lambda path: Path(path).touch() cases = [ (Action.RECV, mock_context.pipes.recv_pipes, mkdir), (Action.SEND, mock_context.pipes.send_pipes, mkdir), - (Action.RECV, mock_context.pipes.recv_pipes, mknod), - (Action.SEND, mock_context.pipes.send_pipes, mknod) + (Action.RECV, mock_context.pipes.recv_pipes, touch), + (Action.SEND, mock_context.pipes.send_pipes, touch) ] for action, pipes, creator in cases: From 552c9f5f93ea205c86b15a2c1c4f8d23b2ad35e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 15 Aug 2019 10:43:30 +0200 Subject: [PATCH 124/174] Rename FrontendHandler -> FrontendProxyHandler --- tfw/components/frontend/__init__.py | 2 +- .../frontend/{frontend_handler.py => frontend_proxy_handler.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename tfw/components/frontend/{frontend_handler.py => frontend_proxy_handler.py} (96%) diff --git a/tfw/components/frontend/__init__.py b/tfw/components/frontend/__init__.py index 3e0b871..4ce9e31 100644 --- a/tfw/components/frontend/__init__.py +++ b/tfw/components/frontend/__init__.py @@ -1,2 +1,2 @@ -from .frontend_handler import FrontendHandler +from .frontend_proxy_handler import FrontendProxyHandler from .message_sender import MessageSender diff --git a/tfw/components/frontend/frontend_handler.py b/tfw/components/frontend/frontend_proxy_handler.py similarity index 96% rename from tfw/components/frontend/frontend_handler.py rename to tfw/components/frontend/frontend_proxy_handler.py index dfb50bf..10d1080 100644 --- a/tfw/components/frontend/frontend_handler.py +++ b/tfw/components/frontend/frontend_proxy_handler.py @@ -3,7 +3,7 @@ from tfw.internals.networking import Scope from .message_storage import FrontendMessageStorage -class FrontendHandler: +class FrontendProxyHandler: keys = ['console', 'dashboard', 'message', 'ide.read', 'recover'] def __init__(self): From c84ac9914164673ce750613cd51b5bd7cf402354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 15 Aug 2019 14:37:08 +0200 Subject: [PATCH 125/174] Write process logs to console with a separate handler --- tfw/components/frontend/__init__.py | 1 + .../frontend/console_logs_handler.py | 19 +++++++++++++++++++ .../process_management/process_handler.py | 17 +++-------------- 3 files changed, 23 insertions(+), 14 deletions(-) create mode 100644 tfw/components/frontend/console_logs_handler.py diff --git a/tfw/components/frontend/__init__.py b/tfw/components/frontend/__init__.py index 4ce9e31..2300690 100644 --- a/tfw/components/frontend/__init__.py +++ b/tfw/components/frontend/__init__.py @@ -1,2 +1,3 @@ from .frontend_proxy_handler import FrontendProxyHandler +from .console_logs_handler import ConsoleLogsHandler from .message_sender import MessageSender diff --git a/tfw/components/frontend/console_logs_handler.py b/tfw/components/frontend/console_logs_handler.py new file mode 100644 index 0000000..c3baf45 --- /dev/null +++ b/tfw/components/frontend/console_logs_handler.py @@ -0,0 +1,19 @@ +import logging + +LOG = logging.getLogger(__name__) + + +class ConsoleLogsHandler: + keys = ['process.log.new'] + + def __init__(self, *, stream): + self.stream = stream + + def handle_event(self, message, connector): + try: + connector.send_message({ + 'key': 'console.write', + 'value': message[self.stream] + }) + except KeyError: + LOG.error('Invalid %s message received: %s', self.keys, message) diff --git a/tfw/components/process_management/process_handler.py b/tfw/components/process_management/process_handler.py index 33e58fa..e24a83f 100644 --- a/tfw/components/process_management/process_handler.py +++ b/tfw/components/process_management/process_handler.py @@ -3,18 +3,16 @@ from xmlrpc.client import Fault as SupervisorFault from tfw.internals.networking import Scope -from .supervisor import ProcessManager, ProcessLogManager +from .supervisor import ProcessManager LOG = logging.getLogger(__name__) -class ProcessHandler(ProcessManager, ProcessLogManager): +class ProcessHandler(ProcessManager): keys = ['process'] - def __init__(self, *, supervisor_uri, log_tail=0): + def __init__(self, *, supervisor_uri): ProcessManager.__init__(self, supervisor_uri) - ProcessLogManager.__init__(self, supervisor_uri) - self.log_tail = log_tail self.commands = { 'process.start': self.start_process, @@ -28,15 +26,6 @@ class ProcessHandler(ProcessManager, ProcessLogManager): self.commands[message['key']](message['name']) except SupervisorFault as fault: message['error'] = fault.faultString - finally: - message['stdout'] = self.read_stdout( - message['name'], - self.log_tail - ) - message['stderr'] = self.read_stderr( - message['name'], - self.log_tail - ) connector.send_message(message, scope=Scope.WEBSOCKET) except KeyError: if not message['key'].startswith('process.log'): From e0a915ac556c4dd3f4fbf67907906d32a5d17f3d Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 15 Aug 2019 15:36:57 +0200 Subject: [PATCH 126/174] Move initial FSM trigger to backend --- tfw/components/frontend/frontend_proxy_handler.py | 4 ++-- tfw/components/fsm/fsm_handler.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tfw/components/frontend/frontend_proxy_handler.py b/tfw/components/frontend/frontend_proxy_handler.py index 10d1080..a46f015 100644 --- a/tfw/components/frontend/frontend_proxy_handler.py +++ b/tfw/components/frontend/frontend_proxy_handler.py @@ -4,7 +4,7 @@ from .message_storage import FrontendMessageStorage class FrontendProxyHandler: - keys = ['console', 'dashboard', 'message', 'ide.read', 'recover'] + keys = ['console', 'dashboard', 'frontend.ready', 'message', 'ide.read'] def __init__(self): self.connector = None @@ -15,7 +15,7 @@ class FrontendProxyHandler: def handle_event(self, message, _): self._frontend_message_storage.save_message(message) - if message['key'] == 'recover': + if message['key'] == 'frontend.ready': self.recover_frontend() if self._filter_message(message): self.send_message(message) diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index 24b8231..b26db86 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -11,16 +11,21 @@ LOG = logging.getLogger(__name__) class FSMHandler: keys = ['fsm'] - def __init__(self, *, fsm_type): + def __init__(self, *, fsm_type, initial_trigger): self.fsm = fsm_type() self._fsm_updater = FSMUpdater(self.fsm) self.auth_key = KeyManager().auth_key + self.initial_trigger = initial_trigger self.command_handlers = { + 'frontend.ready': self.handle_ready, 'fsm.step' : self.handle_step, 'fsm.announce' : self.handle_announce } + def start(self): + self.connector.subscribe('frontend.ready') + def handle_event(self, message, connector): try: message = self.command_handlers[message['key']](message) @@ -31,6 +36,11 @@ class FSMHandler: except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) + def handle_ready(self, message): + self.fsm.step(self.initial_trigger) + self.connector.unsubscribe('frontend.ready') + return message + def handle_step(self, message): if self.fsm.step(message['trigger']): return message From 8bde3191595b0f03ace8a7d16d86c4186d59f6cf Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 15 Aug 2019 17:02:29 +0200 Subject: [PATCH 127/174] Add underscore to process name --- supervisor/components/tfw_server.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervisor/components/tfw_server.conf b/supervisor/components/tfw_server.conf index a131674..de72b69 100644 --- a/supervisor/components/tfw_server.conf +++ b/supervisor/components/tfw_server.conf @@ -1,4 +1,4 @@ -[program:tfwserver] +[program:tfw_server] user=root directory=%(ENV_TFW_SERVER_DIR)s command=python3 -u tfw_server.py From 75a9fce47c7b77b3aa4a52768f8472e95a155aeb Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 15 Aug 2019 17:03:15 +0200 Subject: [PATCH 128/174] Modify resources' directory structure --- Dockerfile | 14 +++++--------- supervisor/components/tfw_init.conf | 6 ++++++ supervisor/tfw_init.sh | 13 +++++++++++++ supervisor/tfw_server.py | 11 ++--------- tfw/main/__init__.py | 3 ++- tfw/main/logging.py | 12 ++++++++++++ 6 files changed, 40 insertions(+), 19 deletions(-) create mode 100644 supervisor/components/tfw_init.conf create mode 100644 supervisor/tfw_init.sh create mode 100644 tfw/main/logging.py diff --git a/Dockerfile b/Dockerfile index 5df5cf4..b3bd3b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,16 +39,14 @@ ENV PYTHONPATH="/usr/local/lib" \ TFW_FRONTEND_DIR="/srv/frontend" \ TFW_DIR="/.tfw" \ TFW_SERVER_DIR="/.tfw/tfw_server" \ - TFW_SNAPSHOTS_DIR="/.tfw/snapshots" \ TFW_AUTH_KEY="/tmp/tfw-auth.key" \ + TFW_LOGS_DIR="/var/log/tfw" \ + TFW_PIPES_DIR="/run/tfw" \ + TFW_SNAPSHOTS_DIR="/tmp/tfw-snapshots" \ TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history" \ - TFW_LOGFILE="/var/log/tfw.log" \ PROMPT_COMMAND="history -a" -COPY bashrc /tmp -RUN echo "export HISTFILE=${TFW_HISTFILE}" >> /tmp/bashrc &&\ - cat /tmp/bashrc >> /home/${AVATAO_USER}/.bashrc - +COPY bashrc supervisor/tfw_init.sh /tmp/ COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF} COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS} COPY nginx/nginx.conf ${TFW_NGINX_CONF} @@ -57,9 +55,7 @@ COPY nginx/components/ ${TFW_NGINX_COMPONENTS} COPY tfw ${TFW_LIB_DIR}/tfw COPY supervisor/tfw_server.py ${TFW_SERVER_DIR}/ -RUN for dir in "${TFW_LIB_DIR}"/tfw "/etc/nginx" "/etc/supervisor"; do \ - chown -R root:root "$dir" && chmod -R 700 "$dir"; \ - done +VOLUME ["${TFW_LOGS_DIR}", "${TFW_PIPES_DIR}"] ONBUILD ARG BUILD_CONTEXT="solvable" ONBUILD ARG NOFRONTEND="" diff --git a/supervisor/components/tfw_init.conf b/supervisor/components/tfw_init.conf new file mode 100644 index 0000000..51f65b5 --- /dev/null +++ b/supervisor/components/tfw_init.conf @@ -0,0 +1,6 @@ +[program:tfw_init] +user=root +directory=/tmp +command=bash tfw_init.sh +autorestart=false +startsecs=0 diff --git a/supervisor/tfw_init.sh b/supervisor/tfw_init.sh new file mode 100644 index 0000000..2006f09 --- /dev/null +++ b/supervisor/tfw_init.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail + +echo "export HISTFILE=\"${TFW_HISTFILE}\"" >> /tmp/bashrc && + cat /tmp/bashrc >> "/home/${AVATAO_USER}/.bashrc" + +if [[ -z "${HOTRELOAD}" ]]; then + for dir in "${TFW_LIB_DIR}/tfw" "/etc/nginx" "/etc/supervisor"; do + chown -R root:root "${dir}" && chmod -R 700 "${dir}"; + done +fi + +rm -f bashrc requirements.txt tfw_init.sh diff --git a/supervisor/tfw_server.py b/supervisor/tfw_server.py index eccd753..d3bbb5e 100644 --- a/supervisor/tfw_server.py +++ b/supervisor/tfw_server.py @@ -1,17 +1,10 @@ -from sys import stderr - from tornado.ioloop import IOLoop -from tfw.main import TFWServer, setup_signal_handlers -from tfw.config import TFWENV -from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter +from tfw.main import TFWServer, setup_logger, setup_signal_handlers if __name__ == '__main__': - Logger([ - Log(stderr, LogFormatter(20)), - Log(TFWENV.LOGFILE, VerboseLogFormatter()) - ]).start() + setup_logger(__file__) TFWServer().listen() setup_signal_handlers() diff --git a/tfw/main/__init__.py b/tfw/main/__init__.py index 3e731f9..9dac792 100644 --- a/tfw/main/__init__.py +++ b/tfw/main/__init__.py @@ -1,4 +1,5 @@ -from .tfw_connector import TFWUplinkConnector, TFWConnector from .event_handler_factory import EventHandlerFactory +from .logging import setup_logger from .signal_handling import setup_signal_handlers +from .tfw_connector import TFWUplinkConnector, TFWConnector from .tfw_server import TFWServer diff --git a/tfw/main/logging.py b/tfw/main/logging.py new file mode 100644 index 0000000..6d7fb02 --- /dev/null +++ b/tfw/main/logging.py @@ -0,0 +1,12 @@ +from sys import stderr +from os.path import join + +from tfw.config import TFWENV +from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter + + +def setup_logger(name): + Logger([ + Log(stderr, LogFormatter(20)), + Log(join(TFWENV.LOGS_DIR, name+'.log'), VerboseLogFormatter()) + ]).start() From 2e9a84bc9b7510443c836bef6bccc911bb58de17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 15 Aug 2019 17:26:23 +0200 Subject: [PATCH 129/174] Fix init script without HOTRELOAD --- supervisor/tfw_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervisor/tfw_init.sh b/supervisor/tfw_init.sh index 2006f09..4b38337 100644 --- a/supervisor/tfw_init.sh +++ b/supervisor/tfw_init.sh @@ -4,7 +4,7 @@ set -euo pipefail echo "export HISTFILE=\"${TFW_HISTFILE}\"" >> /tmp/bashrc && cat /tmp/bashrc >> "/home/${AVATAO_USER}/.bashrc" -if [[ -z "${HOTRELOAD}" ]]; then +if [[ -z "${HOTRELOAD-}" ]]; then for dir in "${TFW_LIB_DIR}/tfw" "/etc/nginx" "/etc/supervisor"; do chown -R root:root "${dir}" && chmod -R 700 "${dir}"; done From f626fef8f8c20dcdf8bad5ae0b7b328ad2d090b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 15 Aug 2019 17:41:05 +0200 Subject: [PATCH 130/174] Open up TFW pipes --- supervisor/tfw_init.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/supervisor/tfw_init.sh b/supervisor/tfw_init.sh index 4b38337..f2ecd72 100644 --- a/supervisor/tfw_init.sh +++ b/supervisor/tfw_init.sh @@ -10,4 +10,6 @@ if [[ -z "${HOTRELOAD-}" ]]; then done fi +chmod 777 "${TFW_PIPES_DIR}" + rm -f bashrc requirements.txt tfw_init.sh From 2e5867cc4975490fc6a0b982a947f6d03c6bad8c Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 23 Aug 2019 15:27:03 +0200 Subject: [PATCH 131/174] Introduce intent for TFW messages and fix FSM related bugs --- tfw/components/fsm/fsm_handler.py | 15 ++++++++------- tfw/components/fsm/fsm_updater.py | 3 ++- .../process_management/log_inotify_observer.py | 6 ++---- .../process_management/process_handler.py | 4 ++-- tfw/event_handlers.py | 2 +- tfw/fsm/fsm_base.py | 2 +- tfw/internals/crypto/authentication.py | 2 ++ tfw/internals/event_handling/__init__.py | 1 + .../event_handling/control_event_handler.py | 9 +++++++++ tfw/internals/event_handling/fsm_aware.py | 1 + .../event_handling/signed_event_handler.py | 4 ++-- tfw/internals/networking/__init__.py | 1 + tfw/internals/networking/intent.py | 6 ++++++ tfw/internals/networking/zmq_connector.py | 5 ++++- 14 files changed, 42 insertions(+), 19 deletions(-) create mode 100644 tfw/internals/event_handling/control_event_handler.py create mode 100644 tfw/internals/networking/intent.py diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index b26db86..9ea1374 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -1,7 +1,7 @@ import logging from tfw.internals.crypto import KeyManager, sign_message -from tfw.internals.networking import Scope +from tfw.internals.networking import Scope, Intent from .fsm_updater import FSMUpdater @@ -12,15 +12,16 @@ class FSMHandler: keys = ['fsm'] def __init__(self, *, fsm_type, initial_trigger): + self.connector = None self.fsm = fsm_type() self._fsm_updater = FSMUpdater(self.fsm) self.auth_key = KeyManager().auth_key self.initial_trigger = initial_trigger self.command_handlers = { - 'frontend.ready': self.handle_ready, - 'fsm.step' : self.handle_step, - 'fsm.announce' : self.handle_announce + 'frontend.ready' : self.handle_ready, + 'fsm.step' : self.handle_step, + 'fsm.update' : self.handle_update } def start(self): @@ -32,7 +33,7 @@ class FSMHandler: if message: fsm_update_message = self._fsm_updater.fsm_update sign_message(self.auth_key, fsm_update_message) - connector.send_message(fsm_update_message, Scope.BROADCAST) + connector.send_message(fsm_update_message, Scope.BROADCAST, Intent.EVENT) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) @@ -41,10 +42,10 @@ class FSMHandler: self.connector.unsubscribe('frontend.ready') return message - def handle_step(self, message): + def handle_step(self, message): # pylint: disable=inconsistent-return-statements if self.fsm.step(message['trigger']): return message - def handle_announce(self, message): + def handle_update(self, message): # pylint: disable=no-self-use return message diff --git a/tfw/components/fsm/fsm_updater.py b/tfw/components/fsm/fsm_updater.py index 276b65d..1c671ad 100644 --- a/tfw/components/fsm/fsm_updater.py +++ b/tfw/components/fsm/fsm_updater.py @@ -15,8 +15,9 @@ class FSMUpdater: {'trigger': trigger} for trigger in self.fsm.get_triggers(self.fsm.state) ] + if not self.fsm.event_log: + return {} last_fsm_event = self.fsm.event_log[-1] - last_fsm_event['timestamp'] = last_fsm_event['timestamp'].isoformat() return { 'current_state': self.fsm.state, 'valid_transitions': valid_transitions, diff --git a/tfw/components/process_management/log_inotify_observer.py b/tfw/components/process_management/log_inotify_observer.py index 1c80dd1..1a5887a 100644 --- a/tfw/components/process_management/log_inotify_observer.py +++ b/tfw/components/process_management/log_inotify_observer.py @@ -1,6 +1,4 @@ -import logging - -from tfw.internals.networking import Scope +from tfw.internals.networking import Scope, Intent from tfw.internals.inotify import InotifyObserver from .supervisor import ProcessLogManager @@ -33,4 +31,4 @@ class LogInotifyObserver(InotifyObserver, ProcessLogManager): 'key': 'process.log.new', 'stdout': self.read_stdout(self.process_name, tail=self.log_tail), 'stderr': self.read_stderr(self.process_name, tail=self.log_tail) - }, Scope.BROADCAST) + }, Scope.BROADCAST, Intent.EVENT) diff --git a/tfw/components/process_management/process_handler.py b/tfw/components/process_management/process_handler.py index e24a83f..04c121b 100644 --- a/tfw/components/process_management/process_handler.py +++ b/tfw/components/process_management/process_handler.py @@ -1,7 +1,7 @@ import logging from xmlrpc.client import Fault as SupervisorFault -from tfw.internals.networking import Scope +from tfw.internals.networking import Scope, Intent from .supervisor import ProcessManager @@ -26,7 +26,7 @@ class ProcessHandler(ProcessManager): self.commands[message['key']](message['name']) except SupervisorFault as fault: message['error'] = fault.faultString - connector.send_message(message, scope=Scope.WEBSOCKET) + connector.send_message(message, scope=Scope.BROADCAST, intent=Intent.EVENT) except KeyError: if not message['key'].startswith('process.log'): LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/tfw/event_handlers.py b/tfw/event_handlers.py index 0ea1050..3cd92f5 100644 --- a/tfw/event_handlers.py +++ b/tfw/event_handlers.py @@ -1,2 +1,2 @@ # pylint: disable=unused-import -from tfw.internals.event_handling import EventHandler, FSMAwareEventHandler, SignedEventHandler +from tfw.internals.event_handling import EventHandler, ControlEventHandler, FSMAwareEventHandler, SignedEventHandler diff --git a/tfw/fsm/fsm_base.py b/tfw/fsm/fsm_base.py index 0fac40b..c35d094 100644 --- a/tfw/fsm/fsm_base.py +++ b/tfw/fsm/fsm_base.py @@ -76,7 +76,7 @@ class FSMBase(Machine, CallbackMixin): 'from_state': from_state, 'to_state': self.state, 'trigger': trigger, - 'timestamp': datetime.utcnow() + 'timestamp': datetime.utcnow().isoformat() }) @property diff --git a/tfw/internals/crypto/authentication.py b/tfw/internals/crypto/authentication.py index 26e9e11..0ca19a3 100644 --- a/tfw/internals/crypto/authentication.py +++ b/tfw/internals/crypto/authentication.py @@ -12,6 +12,7 @@ from tfw.internals.networking import message_bytes def sign_message(key, message): message.pop('scope', None) + message.pop('intent', None) message.pop('signature', None) signature = message_signature(key, message) message['signature'] = b64encode(signature).decode() @@ -23,6 +24,7 @@ def message_signature(key, message): def verify_message(key, message): message.pop('scope', None) + message.pop('intent', None) message = deepcopy(message) try: signature_b64 = message.pop('signature') diff --git a/tfw/internals/event_handling/__init__.py b/tfw/internals/event_handling/__init__.py index 8238d21..0803cc5 100644 --- a/tfw/internals/event_handling/__init__.py +++ b/tfw/internals/event_handling/__init__.py @@ -1,4 +1,5 @@ from .event_handler import EventHandler from .event_handler_factory_base import EventHandlerFactoryBase +from .control_event_handler import ControlEventHandler from .fsm_aware_event_handler import FSMAwareEventHandler from .signed_event_handler import SignedEventHandler diff --git a/tfw/internals/event_handling/control_event_handler.py b/tfw/internals/event_handling/control_event_handler.py new file mode 100644 index 0000000..dda4dae --- /dev/null +++ b/tfw/internals/event_handling/control_event_handler.py @@ -0,0 +1,9 @@ +from tfw.internals.networking import Intent + +from .event_handler import EventHandler + + +class ControlEventHandler(EventHandler): # pylint: disable=abstract-method + def _event_callback(self, message): + if message.get('intent') != Intent.EVENT.value: + self.handle_event(message, self.connector) diff --git a/tfw/internals/event_handling/fsm_aware.py b/tfw/internals/event_handling/fsm_aware.py index b888220..e5de2dc 100644 --- a/tfw/internals/event_handling/fsm_aware.py +++ b/tfw/internals/event_handling/fsm_aware.py @@ -7,6 +7,7 @@ LOG = logging.getLogger(__name__) class FSMAware: keys = ['fsm.update'] + def __init__(self): self.fsm_state = None self.fsm_in_accepted_state = False diff --git a/tfw/internals/event_handling/signed_event_handler.py b/tfw/internals/event_handling/signed_event_handler.py index 1ee72b8..581035f 100644 --- a/tfw/internals/event_handling/signed_event_handler.py +++ b/tfw/internals/event_handling/signed_event_handler.py @@ -2,13 +2,13 @@ import logging from tfw.internals.crypto import KeyManager, verify_message -from .event_handler import EventHandler +from .control_event_handler import ControlEventHandler LOG = logging.getLogger(__name__) # pylint: disable=abstract-method -class SignedEventHandler(EventHandler): +class SignedEventHandler(ControlEventHandler): def __init__(self, connector): self._auth_key = KeyManager().auth_key super().__init__(connector) diff --git a/tfw/internals/networking/__init__.py b/tfw/internals/networking/__init__.py index dca7b24..520cbec 100644 --- a/tfw/internals/networking/__init__.py +++ b/tfw/internals/networking/__init__.py @@ -2,3 +2,4 @@ from .serialization import serialize_tfw_msg, deserialize_tfw_msg, with_deserial from .zmq_connector import ZMQConnector, ZMQDownlinkConnector, ZMQUplinkConnector from .zmq_listener import ZMQListener from .scope import Scope +from .intent import Intent diff --git a/tfw/internals/networking/intent.py b/tfw/internals/networking/intent.py new file mode 100644 index 0000000..a13c7bb --- /dev/null +++ b/tfw/internals/networking/intent.py @@ -0,0 +1,6 @@ +from enum import Enum + + +class Intent(Enum): + CONTROL = 'control' + EVENT = 'event' diff --git a/tfw/internals/networking/zmq_connector.py b/tfw/internals/networking/zmq_connector.py index 99920c7..10c5114 100644 --- a/tfw/internals/networking/zmq_connector.py +++ b/tfw/internals/networking/zmq_connector.py @@ -4,6 +4,7 @@ import zmq from zmq.eventloop.zmqstream import ZMQStream from .scope import Scope +from .intent import Intent from .serialization import ( serialize_tfw_msg, deserialize_tfw_msg, @@ -51,8 +52,10 @@ class ZMQUplinkConnector: self._zmq_push_socket.setsockopt(zmq.SNDHWM, 0) self._zmq_push_socket.connect(connect_addr) - def send_message(self, message, scope=Scope.ZMQ): + def send_message(self, message, scope=Scope.ZMQ, intent=None): message['scope'] = scope.value + if isinstance(intent, Intent): + message['intent'] = intent.value self._zmq_push_socket.send_multipart(serialize_tfw_msg(message)) def close(self): From 965973a32fc969fbee47f051db061f1c9e80415e Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 26 Aug 2019 11:10:39 +0200 Subject: [PATCH 132/174] Fix issues in PR #64 --- tfw/components/fsm/fsm_handler.py | 4 ++-- tfw/components/fsm/fsm_updater.py | 2 +- tfw/event_handlers.py | 2 +- tfw/internals/event_handling/__init__.py | 1 + .../event_handling/control_event_handler.py | 5 ++--- tfw/internals/event_handling/event_handler.py | 6 +++++- .../event_handling/signed_control_event_handler.py | 9 +++++++++ tfw/internals/event_handling/signed_event_handler.py | 12 ++++++------ tfw/internals/networking/zmq_connector.py | 2 +- 9 files changed, 28 insertions(+), 15 deletions(-) create mode 100644 tfw/internals/event_handling/signed_control_event_handler.py diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index 9ea1374..ae9a171 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -27,13 +27,13 @@ class FSMHandler: def start(self): self.connector.subscribe('frontend.ready') - def handle_event(self, message, connector): + def handle_event(self, message, _): try: message = self.command_handlers[message['key']](message) if message: fsm_update_message = self._fsm_updater.fsm_update sign_message(self.auth_key, fsm_update_message) - connector.send_message(fsm_update_message, Scope.BROADCAST, Intent.EVENT) + self.connector.send_message(fsm_update_message, Scope.BROADCAST, Intent.EVENT) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/tfw/components/fsm/fsm_updater.py b/tfw/components/fsm/fsm_updater.py index 1c671ad..8ab488f 100644 --- a/tfw/components/fsm/fsm_updater.py +++ b/tfw/components/fsm/fsm_updater.py @@ -16,7 +16,7 @@ class FSMUpdater: for trigger in self.fsm.get_triggers(self.fsm.state) ] if not self.fsm.event_log: - return {} + return None last_fsm_event = self.fsm.event_log[-1] return { 'current_state': self.fsm.state, diff --git a/tfw/event_handlers.py b/tfw/event_handlers.py index 3cd92f5..a804bc4 100644 --- a/tfw/event_handlers.py +++ b/tfw/event_handlers.py @@ -1,2 +1,2 @@ # pylint: disable=unused-import -from tfw.internals.event_handling import EventHandler, ControlEventHandler, FSMAwareEventHandler, SignedEventHandler +from tfw.internals.event_handling import EventHandler, ControlEventHandler, FSMAwareEventHandler, SignedEventHandler, SignedControlEventHandler diff --git a/tfw/internals/event_handling/__init__.py b/tfw/internals/event_handling/__init__.py index 0803cc5..bd5e14f 100644 --- a/tfw/internals/event_handling/__init__.py +++ b/tfw/internals/event_handling/__init__.py @@ -3,3 +3,4 @@ from .event_handler_factory_base import EventHandlerFactoryBase from .control_event_handler import ControlEventHandler from .fsm_aware_event_handler import FSMAwareEventHandler from .signed_event_handler import SignedEventHandler +from .signed_control_event_handler import SignedControlEventHandler diff --git a/tfw/internals/event_handling/control_event_handler.py b/tfw/internals/event_handling/control_event_handler.py index dda4dae..381963e 100644 --- a/tfw/internals/event_handling/control_event_handler.py +++ b/tfw/internals/event_handling/control_event_handler.py @@ -4,6 +4,5 @@ from .event_handler import EventHandler class ControlEventHandler(EventHandler): # pylint: disable=abstract-method - def _event_callback(self, message): - if message.get('intent') != Intent.EVENT.value: - self.handle_event(message, self.connector) + def _validate_message(self, message): + return message.get('intent') != Intent.EVENT.value diff --git a/tfw/internals/event_handling/event_handler.py b/tfw/internals/event_handling/event_handler.py index f288042..fd7c69c 100644 --- a/tfw/internals/event_handling/event_handler.py +++ b/tfw/internals/event_handling/event_handler.py @@ -9,7 +9,11 @@ class EventHandler: self.connector.register_callback(self._event_callback) def _event_callback(self, message): - self.handle_event(message, self.connector) + if self._validate_message(message): + self.handle_event(message, self.connector) + + def _validate_message(self, message): + return True def handle_event(self, message, connector): raise NotImplementedError() diff --git a/tfw/internals/event_handling/signed_control_event_handler.py b/tfw/internals/event_handling/signed_control_event_handler.py new file mode 100644 index 0000000..5375280 --- /dev/null +++ b/tfw/internals/event_handling/signed_control_event_handler.py @@ -0,0 +1,9 @@ +from .control_event_handler import ControlEventHandler +from .signed_event_handler import SignedEventHandler + +class SignedControlEventHandler(ControlEventHandler, SignedEventHandler): + def _validate_message(self, message): + return ( + ControlEventHandler._validate_message(self, message) and + SignedEventHandler._validate_message(self, message) + ) diff --git a/tfw/internals/event_handling/signed_event_handler.py b/tfw/internals/event_handling/signed_event_handler.py index 581035f..15f4439 100644 --- a/tfw/internals/event_handling/signed_event_handler.py +++ b/tfw/internals/event_handling/signed_event_handler.py @@ -2,19 +2,19 @@ import logging from tfw.internals.crypto import KeyManager, verify_message -from .control_event_handler import ControlEventHandler +from .event_handler import EventHandler LOG = logging.getLogger(__name__) # pylint: disable=abstract-method -class SignedEventHandler(ControlEventHandler): +class SignedEventHandler(EventHandler): def __init__(self, connector): self._auth_key = KeyManager().auth_key super().__init__(connector) - def _event_callback(self, message): - if verify_message(self._auth_key, message): - self.handle_event(message, self.connector) - else: + def _validate_message(self, message): + is_valid = verify_message(self._auth_key, message) + if not is_valid: LOG.error('Message does not have valid signature: %s', message) + return is_valid diff --git a/tfw/internals/networking/zmq_connector.py b/tfw/internals/networking/zmq_connector.py index 10c5114..ff82758 100644 --- a/tfw/internals/networking/zmq_connector.py +++ b/tfw/internals/networking/zmq_connector.py @@ -54,7 +54,7 @@ class ZMQUplinkConnector: def send_message(self, message, scope=Scope.ZMQ, intent=None): message['scope'] = scope.value - if isinstance(intent, Intent): + if intent is not None: message['intent'] = intent.value self._zmq_push_socket.send_multipart(serialize_tfw_msg(message)) From 0f379891e35c8a4c2db964dce16cdad58b9ba602 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 23 Aug 2019 16:15:27 +0200 Subject: [PATCH 133/174] Move message queue logic to backend --- tfw/components/frontend/__init__.py | 3 +- .../frontend/frontend_proxy_handler.py | 2 +- .../message_queue_handler/__init__.py | 1 + .../message_queue_handler.py | 50 ++++++++++++++ .../test_message_queue.py | 67 +++++++++++++++++++ tfw/components/frontend/message_sender.py | 8 --- tfw/components/frontend/message_storage.py | 4 -- 7 files changed, 121 insertions(+), 14 deletions(-) create mode 100644 tfw/components/frontend/message_queue_handler/__init__.py create mode 100644 tfw/components/frontend/message_queue_handler/message_queue_handler.py create mode 100644 tfw/components/frontend/message_queue_handler/test_message_queue.py diff --git a/tfw/components/frontend/__init__.py b/tfw/components/frontend/__init__.py index 2300690..3dab104 100644 --- a/tfw/components/frontend/__init__.py +++ b/tfw/components/frontend/__init__.py @@ -1,3 +1,4 @@ -from .frontend_proxy_handler import FrontendProxyHandler from .console_logs_handler import ConsoleLogsHandler +from .frontend_proxy_handler import FrontendProxyHandler +from .message_queue_handler import MessageQueueHandler from .message_sender import MessageSender diff --git a/tfw/components/frontend/frontend_proxy_handler.py b/tfw/components/frontend/frontend_proxy_handler.py index a46f015..e7c7b06 100644 --- a/tfw/components/frontend/frontend_proxy_handler.py +++ b/tfw/components/frontend/frontend_proxy_handler.py @@ -22,7 +22,7 @@ class FrontendProxyHandler: @staticmethod def _filter_message(message): - return not message['key'].startswith('ide') + return not message['key'].startswith(('ide', 'message.queue')) def recover_frontend(self): for message in self._frontend_message_storage.messages: diff --git a/tfw/components/frontend/message_queue_handler/__init__.py b/tfw/components/frontend/message_queue_handler/__init__.py new file mode 100644 index 0000000..0eb2a68 --- /dev/null +++ b/tfw/components/frontend/message_queue_handler/__init__.py @@ -0,0 +1 @@ +from .message_queue_handler import MessageQueueHandler diff --git a/tfw/components/frontend/message_queue_handler/message_queue_handler.py b/tfw/components/frontend/message_queue_handler/message_queue_handler.py new file mode 100644 index 0000000..80493b2 --- /dev/null +++ b/tfw/components/frontend/message_queue_handler/message_queue_handler.py @@ -0,0 +1,50 @@ +from time import sleep +from queue import Queue +from threading import Thread + + +class MessageQueueHandler: + keys = ['message.queue'] + + def __init__(self, wpm): + self.connector = None + self._wpm, self._cps = None, None + self.wpm = wpm + self._queue = Queue() + self._thread = Thread(target=self.dispatch_messages) + + @property + def wpm(self): + return self._wpm + + @wpm.setter + def wpm(self, wpm): + self._wpm = wpm + self._cps = 5*wpm/60 + + def dispatch_messages(self): + for message in iter(self._queue.get, None): + sleep(len(message['message'])/self._cps) + self.connector.send_message(message) + + def handle_event(self, message, _): + for unpacked in self.generate_messages_from_queue(message): + self._queue.put(unpacked) + + @staticmethod + def generate_messages_from_queue(queue_message): + last = queue_message['value'][-1] + for message in queue_message['value']: + yield { + 'key': 'message.send', + 'typing': message is not last, + **message + } + + def start(self): + self._thread.start() + + def cleanup(self): + self._queue.queue.clear() + self._queue.put(None) + self._thread.join() diff --git a/tfw/components/frontend/message_queue_handler/test_message_queue.py b/tfw/components/frontend/message_queue_handler/test_message_queue.py new file mode 100644 index 0000000..ae09b24 --- /dev/null +++ b/tfw/components/frontend/message_queue_handler/test_message_queue.py @@ -0,0 +1,67 @@ +# pylint: disable=redefined-outer-name +from math import inf +from time import sleep +from os import urandom +from random import randint + +import pytest + +from .message_queue_handler import MessageQueueHandler + + +class MockConnector: + def __init__(self): + self.callback = None + self.messages = [] + + def raise_event(self, message): + self.callback(message, self) + sleep(0.01) + + def send_message(self, message): + self.messages.append(message) + + +@pytest.fixture +def handler(): + connector = MockConnector() + handler = MessageQueueHandler(inf) + handler.connector = connector + connector.callback = handler.handle_event + handler.start() + yield handler + handler.cleanup() + + +@pytest.fixture +def queue(): + yield { + 'key': 'message.queue', + 'value': [ + {'originator': urandom(4).hex(), 'message': urandom(16).hex()} + for _ in range(randint(5, 10)) + ] + } + + +def test_message_order(handler, queue): + handler.connector.raise_event(queue) + old_list = queue['value'] + new_list = handler.connector.messages + length = len(old_list) + assert len(new_list) == length + for i in range(length): + unpacked = new_list[i] + assert unpacked['key'] == 'message.send' + assert unpacked['originator'] == old_list[i]['originator'] + assert unpacked['typing'] == (i < length-1) + + +def test_wpm(handler, queue): + handler.wpm = 10000 + handler.connector.raise_event(queue) + assert not handler.connector.messages + handler.wpm = 100000000 + handler.connector.raise_event(queue) + sleep(0.25) + assert len(handler.connector.messages) == 2*len(queue['value']) diff --git a/tfw/components/frontend/message_sender.py b/tfw/components/frontend/message_sender.py index 93826df..0f4f249 100644 --- a/tfw/components/frontend/message_sender.py +++ b/tfw/components/frontend/message_sender.py @@ -28,11 +28,3 @@ class MessageSender: 'key': 'message.config', 'originator': originator }) - - @staticmethod - def generate_messages_from_queue(queue_message): - for message in queue_message['value']: - yield { - 'key': 'message.send', - **message - } diff --git a/tfw/components/frontend/message_storage.py b/tfw/components/frontend/message_storage.py index 78bba26..b9ea946 100644 --- a/tfw/components/frontend/message_storage.py +++ b/tfw/components/frontend/message_storage.py @@ -1,8 +1,6 @@ from abc import ABC, abstractmethod from contextlib import suppress -from .message_sender import MessageSender - class MessageStorage(ABC): def __init__(self): @@ -36,13 +34,11 @@ class FrontendMessageStorage(MessageStorage): 'dashboard.terminalMenuItem', 'message.config', 'message.send', - 'message.queue', 'ide.read' )) def _transform_message(self, message): transformations = { - 'message.queue': MessageSender.generate_messages_from_queue, 'ide.read': self._delete_ide_content } if message['key'] in transformations: From c5afdfb1b2abad3bd98515e61ff3b663c285a115 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 23 Aug 2019 16:24:47 +0200 Subject: [PATCH 134/174] Create handler for deploying in the IDE --- .../frontend/frontend_proxy_handler.py | 2 +- tfw/components/ide/__init__.py | 1 + tfw/components/ide/deploy_handler.py | 29 +++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 tfw/components/ide/deploy_handler.py diff --git a/tfw/components/frontend/frontend_proxy_handler.py b/tfw/components/frontend/frontend_proxy_handler.py index e7c7b06..e0a3dc1 100644 --- a/tfw/components/frontend/frontend_proxy_handler.py +++ b/tfw/components/frontend/frontend_proxy_handler.py @@ -4,7 +4,7 @@ from .message_storage import FrontendMessageStorage class FrontendProxyHandler: - keys = ['console', 'dashboard', 'frontend.ready', 'message', 'ide.read'] + keys = ['console', 'dashboard', 'frontend.ready', 'message', 'ide.read', 'deploy.finish'] def __init__(self): self.connector = None diff --git a/tfw/components/ide/__init__.py b/tfw/components/ide/__init__.py index 2d4ae3e..81b35cf 100644 --- a/tfw/components/ide/__init__.py +++ b/tfw/components/ide/__init__.py @@ -1 +1,2 @@ from .ide_handler import IdeHandler +from .deploy_handler import DeployHandler diff --git a/tfw/components/ide/deploy_handler.py b/tfw/components/ide/deploy_handler.py new file mode 100644 index 0000000..ed9886e --- /dev/null +++ b/tfw/components/ide/deploy_handler.py @@ -0,0 +1,29 @@ +class DeployHandler: + keys = ['deploy.start', 'process.restart'] + + def __init__(self, process_name='webservice'): + self.process_name = process_name + + self.commands = { + 'deploy.start': self.handle_deploy, + 'process.restart': self.handle_process + } + + def handle_event(self, message, _): + try: + self.commands[message['key']](message) + except KeyError: + LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) + + def handle_deploy(self, message): + self.connector.send_message({ + 'key': 'process.restart', + 'name': self.process_name + }) + + def handle_process(self, message): + self.connector.send_message({ + 'key': 'deploy.finish', + 'status': 'success' if 'error' not in message else 'error' + }) + From 7a14928432902e593bbcf9e8cb56b7e04794c1ff Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 26 Aug 2019 13:13:15 +0200 Subject: [PATCH 135/174] Refactor deploy handler --- tfw/components/ide/deploy_handler.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tfw/components/ide/deploy_handler.py b/tfw/components/ide/deploy_handler.py index ed9886e..25b035d 100644 --- a/tfw/components/ide/deploy_handler.py +++ b/tfw/components/ide/deploy_handler.py @@ -1,3 +1,8 @@ +import logging + +LOG = logging.getLogger(__name__) + + class DeployHandler: keys = ['deploy.start', 'process.restart'] @@ -22,8 +27,7 @@ class DeployHandler: }) def handle_process(self, message): - self.connector.send_message({ - 'key': 'deploy.finish', - 'status': 'success' if 'error' not in message else 'error' - }) - + response = {'key': 'deploy.finish'} + if 'error' in message: + response['error'] = message['error'] + self.connector.send_message(response) From af3781adb3120ef96214685cee915b4680805d0d Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 26 Aug 2019 13:51:28 +0200 Subject: [PATCH 136/174] Fix issues in PR #67 --- .../message_queue_handler/message_queue_handler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tfw/components/frontend/message_queue_handler/message_queue_handler.py b/tfw/components/frontend/message_queue_handler/message_queue_handler.py index 80493b2..3fbe07f 100644 --- a/tfw/components/frontend/message_queue_handler/message_queue_handler.py +++ b/tfw/components/frontend/message_queue_handler/message_queue_handler.py @@ -11,7 +11,7 @@ class MessageQueueHandler: self._wpm, self._cps = None, None self.wpm = wpm self._queue = Queue() - self._thread = Thread(target=self.dispatch_messages) + self._thread = Thread(target=self._dispatch_messages) @property def wpm(self): @@ -20,19 +20,19 @@ class MessageQueueHandler: @wpm.setter def wpm(self, wpm): self._wpm = wpm - self._cps = 5*wpm/60 + self._cps = 5 * wpm / 60 - def dispatch_messages(self): + def _dispatch_messages(self): for message in iter(self._queue.get, None): sleep(len(message['message'])/self._cps) self.connector.send_message(message) def handle_event(self, message, _): - for unpacked in self.generate_messages_from_queue(message): + for unpacked in self._generate_messages_from_queue(message): self._queue.put(unpacked) @staticmethod - def generate_messages_from_queue(queue_message): + def _generate_messages_from_queue(queue_message): last = queue_message['value'][-1] for message in queue_message['value']: yield { From 0ea6188a052ebea707ad12fbfd06cfe3ee75abc8 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 26 Aug 2019 14:18:23 +0200 Subject: [PATCH 137/174] Make WPM customization available per-message basis --- .../message_queue_handler/message_queue_handler.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tfw/components/frontend/message_queue_handler/message_queue_handler.py b/tfw/components/frontend/message_queue_handler/message_queue_handler.py index 3fbe07f..186caaf 100644 --- a/tfw/components/frontend/message_queue_handler/message_queue_handler.py +++ b/tfw/components/frontend/message_queue_handler/message_queue_handler.py @@ -8,23 +8,15 @@ class MessageQueueHandler: def __init__(self, wpm): self.connector = None - self._wpm, self._cps = None, None self.wpm = wpm self._queue = Queue() self._thread = Thread(target=self._dispatch_messages) - @property - def wpm(self): - return self._wpm - - @wpm.setter - def wpm(self, wpm): - self._wpm = wpm - self._cps = 5 * wpm / 60 - def _dispatch_messages(self): for message in iter(self._queue.get, None): - sleep(len(message['message'])/self._cps) + wpm = message['wpm'] if 'wpm' in message else self.wpm + cps = 5 * wpm / 60 + sleep(len(message['message']) / cps) self.connector.send_message(message) def handle_event(self, message, _): From 947d0a27b723cbeafee1b22696bbd2f07620e2bd Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 26 Aug 2019 14:53:52 +0200 Subject: [PATCH 138/174] History monitor should only emit events --- tfw/components/terminal/history_monitor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tfw/components/terminal/history_monitor.py b/tfw/components/terminal/history_monitor.py index 26b230d..8b34e03 100644 --- a/tfw/components/terminal/history_monitor.py +++ b/tfw/components/terminal/history_monitor.py @@ -2,6 +2,7 @@ from re import findall from re import compile as compileregex from abc import ABC, abstractmethod +from tfw.internals.networking import Intent from tfw.internals.inotify import InotifyObserver @@ -58,7 +59,7 @@ class HistoryMonitor(ABC, InotifyObserver): self.uplink.send_message({ 'key': f'history.{self.domain}', 'value': command - }) + }, intent=Intent.EVENT) class BashMonitor(HistoryMonitor): From 1f813a2138f6fec4ffd931db465bebdf3fd4e885 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 27 Aug 2019 13:13:33 +0200 Subject: [PATCH 139/174] Add minimal init system to container --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index b3bd3b3..d14cc40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,11 @@ RUN curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - COPY requirements.txt /tmp RUN pip3 install -r /tmp/requirements.txt +RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.18.0/tini-amd64 --output /bin/init &&\ + echo "12d20136605531b09a2c2dac02ccee85e1b874eb322ef6baf7561cd93f93c855 /bin/init" |\ + sha256sum --check --status &&\ + chmod 755 /bin/init + ENV TFW_PUBLIC_PORT=8888 \ TFW_WEB_PORT=4242 \ TFW_LOGIN_APP_PORT=6666 \ @@ -73,4 +78,5 @@ ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn install --frozen-lockfil ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn build --no-progress || : ONBUILD RUN test -z "${NOFRONTEND}" && mv /data/dist ${TFW_FRONTEND_DIR} && rm -rf /data || : +ENTRYPOINT ["/bin/init", "--"] CMD exec supervisord --nodaemon --configuration ${TFW_SUPERVISORD_CONF} From ddb47d696cfca5b179b079fec9361a621eb05c9b Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 27 Aug 2019 13:15:59 +0200 Subject: [PATCH 140/174] Trigger first step in an individual event handler --- tfw/components/frontend/__init__.py | 1 + .../frontend/frontend_ready_handler.py | 39 +++++++++++++++++++ tfw/components/fsm/fsm_handler.py | 12 +----- 3 files changed, 41 insertions(+), 11 deletions(-) create mode 100644 tfw/components/frontend/frontend_ready_handler.py diff --git a/tfw/components/frontend/__init__.py b/tfw/components/frontend/__init__.py index 3dab104..a0bf733 100644 --- a/tfw/components/frontend/__init__.py +++ b/tfw/components/frontend/__init__.py @@ -1,4 +1,5 @@ from .console_logs_handler import ConsoleLogsHandler from .frontend_proxy_handler import FrontendProxyHandler +from .frontend_ready_handler import FrontendReadyHandler from .message_queue_handler import MessageQueueHandler from .message_sender import MessageSender diff --git a/tfw/components/frontend/frontend_ready_handler.py b/tfw/components/frontend/frontend_ready_handler.py new file mode 100644 index 0000000..295d257 --- /dev/null +++ b/tfw/components/frontend/frontend_ready_handler.py @@ -0,0 +1,39 @@ +import logging + +from tfw.internals.crypto import KeyManager, sign_message + +LOG = logging.getLogger(__name__) + + +class FrontendReadyHandler: + keys = ['frontend.ready', 'fsm.update'] + + def __init__(self, initial_trigger): + self.connector = None + self._auth_key = KeyManager().auth_key + self.initial_trigger = initial_trigger + + self.commands = { + 'frontend.ready': self.handle_ready, + 'fsm.update': self.handle_update + } + + def handle_event(self, message, _): + try: + self.commands[message['key']]() + except KeyError: + LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) + + def handle_ready(self): + trigger = { + 'key': 'fsm.step', + 'trigger': self.initial_trigger + } + sign_message(self._auth_key, trigger) + self.connector.send_message(trigger) + + def handle_update(self): + self.stop() + + def stop(self): + pass diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index ae9a171..824b69b 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -11,22 +11,17 @@ LOG = logging.getLogger(__name__) class FSMHandler: keys = ['fsm'] - def __init__(self, *, fsm_type, initial_trigger): + def __init__(self, *, fsm_type): self.connector = None self.fsm = fsm_type() self._fsm_updater = FSMUpdater(self.fsm) self.auth_key = KeyManager().auth_key - self.initial_trigger = initial_trigger self.command_handlers = { - 'frontend.ready' : self.handle_ready, 'fsm.step' : self.handle_step, 'fsm.update' : self.handle_update } - def start(self): - self.connector.subscribe('frontend.ready') - def handle_event(self, message, _): try: message = self.command_handlers[message['key']](message) @@ -37,11 +32,6 @@ class FSMHandler: except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) - def handle_ready(self, message): - self.fsm.step(self.initial_trigger) - self.connector.unsubscribe('frontend.ready') - return message - def handle_step(self, message): # pylint: disable=inconsistent-return-statements if self.fsm.step(message['trigger']): return message From 13e247b0098ef96f3a96d4d521d6f6f6a561a8ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 28 Aug 2019 14:56:31 +0200 Subject: [PATCH 141/174] Proxy all keys prefixed with 'frontend' to frontend --- tfw/components/frontend/frontend_proxy_handler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tfw/components/frontend/frontend_proxy_handler.py b/tfw/components/frontend/frontend_proxy_handler.py index e0a3dc1..a7a4910 100644 --- a/tfw/components/frontend/frontend_proxy_handler.py +++ b/tfw/components/frontend/frontend_proxy_handler.py @@ -4,7 +4,7 @@ from .message_storage import FrontendMessageStorage class FrontendProxyHandler: - keys = ['console', 'dashboard', 'frontend.ready', 'message', 'ide.read', 'deploy.finish'] + keys = ['console', 'dashboard', 'frontend', 'message', 'ide.read', 'deploy.finish'] def __init__(self): self.connector = None @@ -22,7 +22,10 @@ class FrontendProxyHandler: @staticmethod def _filter_message(message): - return not message['key'].startswith(('ide', 'message.queue')) + return not message['key'].startswith(( + 'ide', + 'message.queue' + )) def recover_frontend(self): for message in self._frontend_message_storage.messages: From 0295baad4739c8b5ed93b15f32bb12bf3f7ea675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 28 Aug 2019 14:56:59 +0200 Subject: [PATCH 142/174] Fix bug where message scope would be replaced with default scope --- tfw/internals/networking/test_networking.py | 63 ++++++++++++++++++++- tfw/internals/networking/zmq_connector.py | 8 ++- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/tfw/internals/networking/test_networking.py b/tfw/internals/networking/test_networking.py index d1d4261..deb86c5 100644 --- a/tfw/internals/networking/test_networking.py +++ b/tfw/internals/networking/test_networking.py @@ -8,7 +8,7 @@ from contextlib import suppress import pytest from tornado.ioloop import IOLoop -from tfw.internals.networking import ZMQListener, ZMQConnector +from tfw.internals.networking import ZMQListener, ZMQConnector, Scope, Intent @pytest.fixture @@ -96,6 +96,67 @@ def test_server_downlink(zmq_listener, zmq_connector, test_messages): assert messages == test_messages +def test_connector_default_scope_is_zmq(zmq_listener, zmq_connector): + messages = [] + zmq_listener.register_callback(messages.append) + + zmq_connector.send_message({'key': 'cica'}) + + run_ioloop_once() + + assert messages[0]['scope'] == Scope.ZMQ.value + + +def test_connector_preserves_scope(zmq_listener, zmq_connector): + messages = [] + zmq_listener.register_callback(messages.append) + + zmq_connector.send_message({'key': 'cica', 'scope': Scope.WEBSOCKET.value}) + + run_ioloop_once() + + assert messages[0]['scope'] == Scope.WEBSOCKET.value + + +def test_connector_scope_overrides_message_scope(zmq_listener, zmq_connector): + messages = [] + zmq_listener.register_callback(messages.append) + + zmq_connector.send_message( + {'key': 'cica', 'scope': Scope.WEBSOCKET.value}, + scope=Scope.ZMQ + ) + + run_ioloop_once() + + assert messages[0]['scope'] == Scope.ZMQ.value + + +def test_connector_adds_intent(zmq_listener, zmq_connector): + messages = [] + zmq_listener.register_callback(messages.append) + + zmq_connector.send_message( + {'key': 'cica'}, + intent=Intent.EVENT + ) + + run_ioloop_once() + + assert messages[0]['intent'] == Intent.EVENT.value + + +def test_connector_preserves_intent(zmq_listener, zmq_connector): + messages = [] + zmq_listener.register_callback(messages.append) + + zmq_connector.send_message({'key': 'cica', 'intent': Intent.EVENT.value}) + + run_ioloop_once() + + assert messages[0]['intent'] == Intent.EVENT.value + + def test_server_uplink(zmq_listener, zmq_connector, test_messages): messages = [] zmq_connector.subscribe('') diff --git a/tfw/internals/networking/zmq_connector.py b/tfw/internals/networking/zmq_connector.py index ff82758..cd1111e 100644 --- a/tfw/internals/networking/zmq_connector.py +++ b/tfw/internals/networking/zmq_connector.py @@ -4,7 +4,6 @@ import zmq from zmq.eventloop.zmqstream import ZMQStream from .scope import Scope -from .intent import Intent from .serialization import ( serialize_tfw_msg, deserialize_tfw_msg, @@ -52,8 +51,11 @@ class ZMQUplinkConnector: self._zmq_push_socket.setsockopt(zmq.SNDHWM, 0) self._zmq_push_socket.connect(connect_addr) - def send_message(self, message, scope=Scope.ZMQ, intent=None): - message['scope'] = scope.value + def send_message(self, message, scope=None, intent=None): + if 'scope' not in message: + message['scope'] = Scope.ZMQ.value + if scope is not None: + message['scope'] = scope.value if intent is not None: message['intent'] = intent.value self._zmq_push_socket.send_multipart(serialize_tfw_msg(message)) From 3c84d2f3aa50b77e80a75b83088d108ee92bcf7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 28 Aug 2019 15:04:58 +0200 Subject: [PATCH 143/174] Avoid replaying "frontend.ready" messages --- tfw/components/frontend/frontend_proxy_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tfw/components/frontend/frontend_proxy_handler.py b/tfw/components/frontend/frontend_proxy_handler.py index a7a4910..6a364d2 100644 --- a/tfw/components/frontend/frontend_proxy_handler.py +++ b/tfw/components/frontend/frontend_proxy_handler.py @@ -17,6 +17,7 @@ class FrontendProxyHandler: self._frontend_message_storage.save_message(message) if message['key'] == 'frontend.ready': self.recover_frontend() + return if self._filter_message(message): self.send_message(message) From 996e8e2af770a6e1cedf47fc63f7e7b5fbd4141b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 28 Aug 2019 15:28:48 +0200 Subject: [PATCH 144/174] Remove unused keys from FrontendMessageStore --- tfw/components/frontend/message_storage.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tfw/components/frontend/message_storage.py b/tfw/components/frontend/message_storage.py index b9ea946..1ad85c5 100644 --- a/tfw/components/frontend/message_storage.py +++ b/tfw/components/frontend/message_storage.py @@ -30,9 +30,6 @@ class FrontendMessageStorage(MessageStorage): def _filter_message(self, message): return message['key'].startswith(( 'console.write', - 'dashboard.layout', - 'dashboard.terminalMenuItem', - 'message.config', 'message.send', 'ide.read' )) From 25f54a71b5781822c0223d67ce52b36d6803296e Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 28 Aug 2019 16:31:43 +0200 Subject: [PATCH 145/174] Refactor API message format --- tfw/components/frontend/console_logs_handler.py | 2 +- tfw/components/frontend/frontend_ready_handler.py | 4 ++-- .../frontend/message_queue_handler/message_queue_handler.py | 4 ++-- .../frontend/message_queue_handler/test_message_queue.py | 6 +++--- tfw/components/frontend/message_sender.py | 4 ++-- tfw/components/fsm/fsm_handler.py | 6 +++--- tfw/components/terminal/history_monitor.py | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tfw/components/frontend/console_logs_handler.py b/tfw/components/frontend/console_logs_handler.py index c3baf45..eb02aa4 100644 --- a/tfw/components/frontend/console_logs_handler.py +++ b/tfw/components/frontend/console_logs_handler.py @@ -13,7 +13,7 @@ class ConsoleLogsHandler: try: connector.send_message({ 'key': 'console.write', - 'value': message[self.stream] + 'content': message[self.stream] }) except KeyError: LOG.error('Invalid %s message received: %s', self.keys, message) diff --git a/tfw/components/frontend/frontend_ready_handler.py b/tfw/components/frontend/frontend_ready_handler.py index 295d257..3b19708 100644 --- a/tfw/components/frontend/frontend_ready_handler.py +++ b/tfw/components/frontend/frontend_ready_handler.py @@ -26,8 +26,8 @@ class FrontendReadyHandler: def handle_ready(self): trigger = { - 'key': 'fsm.step', - 'trigger': self.initial_trigger + 'key': 'fsm.trigger', + 'transition': self.initial_trigger } sign_message(self._auth_key, trigger) self.connector.send_message(trigger) diff --git a/tfw/components/frontend/message_queue_handler/message_queue_handler.py b/tfw/components/frontend/message_queue_handler/message_queue_handler.py index 186caaf..b60dc6a 100644 --- a/tfw/components/frontend/message_queue_handler/message_queue_handler.py +++ b/tfw/components/frontend/message_queue_handler/message_queue_handler.py @@ -25,8 +25,8 @@ class MessageQueueHandler: @staticmethod def _generate_messages_from_queue(queue_message): - last = queue_message['value'][-1] - for message in queue_message['value']: + last = queue_message['messages'][-1] + for message in queue_message['messages']: yield { 'key': 'message.send', 'typing': message is not last, diff --git a/tfw/components/frontend/message_queue_handler/test_message_queue.py b/tfw/components/frontend/message_queue_handler/test_message_queue.py index ae09b24..198c729 100644 --- a/tfw/components/frontend/message_queue_handler/test_message_queue.py +++ b/tfw/components/frontend/message_queue_handler/test_message_queue.py @@ -37,7 +37,7 @@ def handler(): def queue(): yield { 'key': 'message.queue', - 'value': [ + 'messages': [ {'originator': urandom(4).hex(), 'message': urandom(16).hex()} for _ in range(randint(5, 10)) ] @@ -46,7 +46,7 @@ def queue(): def test_message_order(handler, queue): handler.connector.raise_event(queue) - old_list = queue['value'] + old_list = queue['messages'] new_list = handler.connector.messages length = len(old_list) assert len(new_list) == length @@ -64,4 +64,4 @@ def test_wpm(handler, queue): handler.wpm = 100000000 handler.connector.raise_event(queue) sleep(0.25) - assert len(handler.connector.messages) == 2*len(queue['value']) + assert len(handler.connector.messages) == 2*len(queue['messages']) diff --git a/tfw/components/frontend/message_sender.py b/tfw/components/frontend/message_sender.py index 0f4f249..2987f1e 100644 --- a/tfw/components/frontend/message_sender.py +++ b/tfw/components/frontend/message_sender.py @@ -14,13 +14,13 @@ class MessageSender: def queue_messages(self, messages, originator=None): message_queue = { 'key': 'message.queue', - 'value': [] + 'messages': [] } for message in messages: next_message = {'message': message} if originator: next_message['originator'] = originator - message_queue['value'].append(next_message) + message_queue['messages'].append(next_message) self.uplink.send_message(message_queue) def set_originator(self, originator): diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index 824b69b..02337ad 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -18,8 +18,8 @@ class FSMHandler: self.auth_key = KeyManager().auth_key self.command_handlers = { - 'fsm.step' : self.handle_step, - 'fsm.update' : self.handle_update + 'fsm.trigger': self.handle_step, + 'fsm.update' : self.handle_update } def handle_event(self, message, _): @@ -32,7 +32,7 @@ class FSMHandler: except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) - def handle_step(self, message): # pylint: disable=inconsistent-return-statements + def handle_trigger(self, message): # pylint: disable=inconsistent-return-statements if self.fsm.step(message['trigger']): return message diff --git a/tfw/components/terminal/history_monitor.py b/tfw/components/terminal/history_monitor.py index 8b34e03..7fa4a99 100644 --- a/tfw/components/terminal/history_monitor.py +++ b/tfw/components/terminal/history_monitor.py @@ -58,7 +58,7 @@ class HistoryMonitor(ABC, InotifyObserver): def send_message(self, command): self.uplink.send_message({ 'key': f'history.{self.domain}', - 'value': command + 'command': command }, intent=Intent.EVENT) From c1df007d6c70295cbbc7c490d866ad2e0c352ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 28 Aug 2019 16:34:55 +0200 Subject: [PATCH 146/174] Fix refactored method name --- tfw/components/fsm/fsm_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index 02337ad..729a794 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -18,7 +18,7 @@ class FSMHandler: self.auth_key = KeyManager().auth_key self.command_handlers = { - 'fsm.trigger': self.handle_step, + 'fsm.trigger': self.handle_trigger, 'fsm.update' : self.handle_update } From 016130b0814f9e2932d64838425aeb6bfc3dc2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 28 Aug 2019 16:40:23 +0200 Subject: [PATCH 147/174] Fix pylint issues --- .pylintrc | 2 +- tfw/components/ide/deploy_handler.py | 3 ++- tfw/components/pipe_io/pipe_connector/pipe_connector.py | 2 +- .../pipe_io/pipe_connector/test_proxy_pipe_connector.py | 3 +-- tfw/event_handlers.py | 8 +++++++- tfw/internals/event_handling/event_handler.py | 1 + .../event_handling/signed_control_event_handler.py | 1 + 7 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.pylintrc b/.pylintrc index 961a526..7004a94 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,7 +6,7 @@ disable = missing-docstring, too-few-public-methods, invalid-name [SIMILARITIES] -min-similarity-lines=7 +min-similarity-lines=8 ignore-comments=yes ignore-docstrings=yes ignore-imports=yes diff --git a/tfw/components/ide/deploy_handler.py b/tfw/components/ide/deploy_handler.py index 25b035d..2efd7f6 100644 --- a/tfw/components/ide/deploy_handler.py +++ b/tfw/components/ide/deploy_handler.py @@ -7,6 +7,7 @@ class DeployHandler: keys = ['deploy.start', 'process.restart'] def __init__(self, process_name='webservice'): + self.connector = None self.process_name = process_name self.commands = { @@ -20,7 +21,7 @@ class DeployHandler: except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) - def handle_deploy(self, message): + def handle_deploy(self, _): self.connector.send_message({ 'key': 'process.restart', 'name': self.process_name diff --git a/tfw/components/pipe_io/pipe_connector/pipe_connector.py b/tfw/components/pipe_io/pipe_connector/pipe_connector.py index 1f47ace..820ff3f 100644 --- a/tfw/components/pipe_io/pipe_connector/pipe_connector.py +++ b/tfw/components/pipe_io/pipe_connector/pipe_connector.py @@ -2,7 +2,7 @@ from stat import S_ISFIFO from os import access, mkdir, stat, R_OK from os.path import exists -from pipe_io_server import PipeReaderServer, PipeWriterServer +from pipe_io_server import PipeWriterServer from tfw.internals.inotify import InotifyObserver, InotifyFileCreatedEvent, InotifyFileDeletedEvent diff --git a/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py b/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py index 3961b28..0764d0c 100644 --- a/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py +++ b/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py @@ -8,9 +8,8 @@ from os.path import join from pathlib import Path from tempfile import TemporaryDirectory -from tfw.internals.inotify import InotifyFileCreatedEvent, InotifyFileDeletedEvent - import pytest +from tfw.internals.inotify import InotifyFileCreatedEvent, InotifyFileDeletedEvent from .pipe_connector import PipeConnector from .proxy_pipe_connector_handler import ProxyPipeConnector diff --git a/tfw/event_handlers.py b/tfw/event_handlers.py index a804bc4..9beabe8 100644 --- a/tfw/event_handlers.py +++ b/tfw/event_handlers.py @@ -1,2 +1,8 @@ # pylint: disable=unused-import -from tfw.internals.event_handling import EventHandler, ControlEventHandler, FSMAwareEventHandler, SignedEventHandler, SignedControlEventHandler +from tfw.internals.event_handling import ( + EventHandler, + ControlEventHandler, + FSMAwareEventHandler, + SignedEventHandler, + SignedControlEventHandler +) diff --git a/tfw/internals/event_handling/event_handler.py b/tfw/internals/event_handling/event_handler.py index fd7c69c..d20cd05 100644 --- a/tfw/internals/event_handling/event_handler.py +++ b/tfw/internals/event_handling/event_handler.py @@ -13,6 +13,7 @@ class EventHandler: self.handle_event(message, self.connector) def _validate_message(self, message): + # pylint: disable=unused-argument,no-self-use return True def handle_event(self, message, connector): diff --git a/tfw/internals/event_handling/signed_control_event_handler.py b/tfw/internals/event_handling/signed_control_event_handler.py index 5375280..545a5fc 100644 --- a/tfw/internals/event_handling/signed_control_event_handler.py +++ b/tfw/internals/event_handling/signed_control_event_handler.py @@ -2,6 +2,7 @@ from .control_event_handler import ControlEventHandler from .signed_event_handler import SignedEventHandler class SignedControlEventHandler(ControlEventHandler, SignedEventHandler): + # pylint: disable=abstract-method def _validate_message(self, message): return ( ControlEventHandler._validate_message(self, message) and From 80eca7e322fe5edaae48c7d8474eec1316af3e75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 28 Aug 2019 16:46:05 +0200 Subject: [PATCH 148/174] Fix leftover string from old API --- tfw/components/fsm/fsm_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index 729a794..8f19a3a 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -33,7 +33,7 @@ class FSMHandler: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) def handle_trigger(self, message): # pylint: disable=inconsistent-return-statements - if self.fsm.step(message['trigger']): + if self.fsm.step(message['transition']): return message def handle_update(self, message): From d9bcfb3705d666752c631018bf21b03f2839c3b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 29 Aug 2019 12:03:14 +0200 Subject: [PATCH 149/174] Add metaclass capable of building instances from class name string --- .../type_id_registry/__init__.py | 1 + .../type_id_registry/test_type_id_registry.py | 43 +++++++++++++++++++ .../type_id_registry/type_id_registry.py | 21 +++++++++ 3 files changed, 65 insertions(+) create mode 100644 tfw/internals/event_handling/type_id_registry/__init__.py create mode 100644 tfw/internals/event_handling/type_id_registry/test_type_id_registry.py create mode 100644 tfw/internals/event_handling/type_id_registry/type_id_registry.py diff --git a/tfw/internals/event_handling/type_id_registry/__init__.py b/tfw/internals/event_handling/type_id_registry/__init__.py new file mode 100644 index 0000000..6786ca7 --- /dev/null +++ b/tfw/internals/event_handling/type_id_registry/__init__.py @@ -0,0 +1 @@ +from .type_id_registry import TypeIdREgistryMixin diff --git a/tfw/internals/event_handling/type_id_registry/test_type_id_registry.py b/tfw/internals/event_handling/type_id_registry/test_type_id_registry.py new file mode 100644 index 0000000..617d695 --- /dev/null +++ b/tfw/internals/event_handling/type_id_registry/test_type_id_registry.py @@ -0,0 +1,43 @@ +import pytest + +from .type_id_registry import TypeIdREgistryMixin + + +def test_registry(): + class A(TypeIdREgistryMixin): + _type_id_registry = {} + class B(A): + pass + class C(A): + pass + + # pylint: disable=protected-access + assert A._type_id_registry['B'] == B + assert A._type_id_registry['C'] == C + + +def test_build_by_type_id(): + class D(TypeIdREgistryMixin): + _type_id_registry = {} + built = [] + class E(D): + def __init__(self): + built.append(self) + class F(D): + def __init__(self): + built.append(self) + raise RuntimeError('cica') + class H(E): + pass + + D.build_type('E') + assert isinstance(built[0], E) + + with pytest.raises(RuntimeError) as err: + D.build_type('F') + assert err.value.args[0] == 'cica' + assert isinstance(built[1], F) + + D.build_type('H') + assert isinstance(built[2], H) + assert isinstance(built[2], E) diff --git a/tfw/internals/event_handling/type_id_registry/type_id_registry.py b/tfw/internals/event_handling/type_id_registry/type_id_registry.py new file mode 100644 index 0000000..2737480 --- /dev/null +++ b/tfw/internals/event_handling/type_id_registry/type_id_registry.py @@ -0,0 +1,21 @@ +from abc import ABCMeta +from contextlib import suppress + + +class TypeIdREgistryMeta(ABCMeta): + def __init__(cls, name, bases, attrs): + with suppress(AttributeError): + if cls.__name__ in cls._type_id_registry: + raise RuntimeError('Type id must be unique!') + cls._type_id_registry[cls.__name__] = cls + super().__init__(name, bases, attrs) + + +class TypeIdREgistryMixin(metaclass=TypeIdREgistryMeta): + @classmethod + def build_type(cls, type_id, *args, **kwargs): + try: + instance_type = cls._type_id_registry[type_id] + return instance_type(*args, **kwargs) + except KeyError: + raise RuntimeError(f'No type with id {type_id}!') From 11bb1e492aae3495cc3c3c877fd10580a19234c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 30 Aug 2019 14:44:26 +0200 Subject: [PATCH 150/174] Add get_type() method to TypeIdRegistryMixin --- .../event_handling/type_id_registry/__init__.py | 2 +- .../type_id_registry/test_type_id_registry.py | 13 ++++++++----- .../type_id_registry/type_id_registry.py | 13 ++++++++----- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tfw/internals/event_handling/type_id_registry/__init__.py b/tfw/internals/event_handling/type_id_registry/__init__.py index 6786ca7..45ed6e6 100644 --- a/tfw/internals/event_handling/type_id_registry/__init__.py +++ b/tfw/internals/event_handling/type_id_registry/__init__.py @@ -1 +1 @@ -from .type_id_registry import TypeIdREgistryMixin +from .type_id_registry import TypeIdRegistryMixin diff --git a/tfw/internals/event_handling/type_id_registry/test_type_id_registry.py b/tfw/internals/event_handling/type_id_registry/test_type_id_registry.py index 617d695..df6a994 100644 --- a/tfw/internals/event_handling/type_id_registry/test_type_id_registry.py +++ b/tfw/internals/event_handling/type_id_registry/test_type_id_registry.py @@ -1,23 +1,26 @@ import pytest -from .type_id_registry import TypeIdREgistryMixin +from .type_id_registry import TypeIdRegistryMixin def test_registry(): - class A(TypeIdREgistryMixin): + class A(TypeIdRegistryMixin): _type_id_registry = {} class B(A): pass class C(A): pass + class D(B): + pass # pylint: disable=protected-access - assert A._type_id_registry['B'] == B - assert A._type_id_registry['C'] == C + assert A._type_id_registry['B'] == A.get_type('B') == B + assert A._type_id_registry['C'] == A.get_type('C') == C + assert A._type_id_registry['D'] == A.get_type('D') == D def test_build_by_type_id(): - class D(TypeIdREgistryMixin): + class D(TypeIdRegistryMixin): _type_id_registry = {} built = [] class E(D): diff --git a/tfw/internals/event_handling/type_id_registry/type_id_registry.py b/tfw/internals/event_handling/type_id_registry/type_id_registry.py index 2737480..7970392 100644 --- a/tfw/internals/event_handling/type_id_registry/type_id_registry.py +++ b/tfw/internals/event_handling/type_id_registry/type_id_registry.py @@ -2,7 +2,7 @@ from abc import ABCMeta from contextlib import suppress -class TypeIdREgistryMeta(ABCMeta): +class TypeIdRegistryMeta(ABCMeta): def __init__(cls, name, bases, attrs): with suppress(AttributeError): if cls.__name__ in cls._type_id_registry: @@ -11,11 +11,14 @@ class TypeIdREgistryMeta(ABCMeta): super().__init__(name, bases, attrs) -class TypeIdREgistryMixin(metaclass=TypeIdREgistryMeta): +class TypeIdRegistryMixin(metaclass=TypeIdRegistryMeta): @classmethod - def build_type(cls, type_id, *args, **kwargs): + def get_type(cls, type_id): try: - instance_type = cls._type_id_registry[type_id] - return instance_type(*args, **kwargs) + return cls._type_id_registry[type_id] except KeyError: raise RuntimeError(f'No type with id {type_id}!') + + @classmethod + def build_type(cls, type_id, *args, **kwargs): + return cls.get_type(type_id)(*args, **kwargs) From 25cf6722319cb47b4bf4e44566b284b603a89668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 30 Aug 2019 14:44:57 +0200 Subject: [PATCH 151/174] Enable handlers to specify EventHandler type --- tfw/internals/event_handling/event_handler.py | 6 ++++- .../event_handler_factory_base.py | 15 ++++++++++-- .../event_handling/test_event_handler.py | 24 +++++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/tfw/internals/event_handling/event_handler.py b/tfw/internals/event_handling/event_handler.py index d20cd05..a503649 100644 --- a/tfw/internals/event_handling/event_handler.py +++ b/tfw/internals/event_handling/event_handler.py @@ -1,5 +1,9 @@ -class EventHandler: +from .type_id_registry import TypeIdRegistryMixin + + +class EventHandler(TypeIdRegistryMixin): _instances = set() + _type_id_registry = {} def __init__(self, connector): type(self)._instances.add(self) diff --git a/tfw/internals/event_handling/event_handler_factory_base.py b/tfw/internals/event_handling/event_handler_factory_base.py index b62d9f1..dd76329 100644 --- a/tfw/internals/event_handling/event_handler_factory_base.py +++ b/tfw/internals/event_handling/event_handler_factory_base.py @@ -4,7 +4,7 @@ from .event_handler import EventHandler class EventHandlerFactoryBase: - def build(self, handler_stub, *, keys=None, event_handler_type=EventHandler): + def build(self, handler_stub, *, keys=None, event_handler_type=None): builder = EventHandlerBuilder(handler_stub, keys, event_handler_type) connector = self._build_connector() event_handler = builder.build(connector) @@ -21,7 +21,13 @@ class EventHandlerFactoryBase: class EventHandlerBuilder: def __init__(self, event_handler, supplied_keys, event_handler_type): self._analyzer = HandlerStubAnalyzer(event_handler, supplied_keys) - self._event_handler_type = event_handler_type + self._event_handler_type = self._determine_type(event_handler_type) + + def _determine_type(self, provided_type): + type_ = provided_type or self._analyzer.event_handler_type or EventHandler + if not issubclass(type_, EventHandler): + raise ValueError("No type supplied!") + return type_ def build(self, connector): event_handler = self._event_handler_type(connector) @@ -66,3 +72,8 @@ class HandlerStubAnalyzer: @property def cleanup(self): return self._event_handler.cleanup + + @property + def event_handler_type(self): + with suppress(AttributeError): + return EventHandler.get_type(self._event_handler.type_id) diff --git a/tfw/internals/event_handling/test_event_handler.py b/tfw/internals/event_handling/test_event_handler.py index 6398627..5082ca8 100644 --- a/tfw/internals/event_handling/test_event_handler.py +++ b/tfw/internals/event_handling/test_event_handler.py @@ -6,6 +6,8 @@ import pytest from .event_handler_factory_base import EventHandlerFactoryBase from .event_handler import EventHandler +from .control_event_handler import ControlEventHandler +from .fsm_aware_event_handler import FSMAwareEventHandler class MockEventHandlerFactory(EventHandlerFactoryBase): @@ -130,6 +132,28 @@ def test_build_from_simple_object(test_keys, test_msg): assert msg == test_msg +def test_build_type(): + class SomeControlHandler: + keys = ['cica'] + type_id = 'ControlEventHandler' + # pylint: disable=no-self-use + def handle_event(self, *_): + pass + + eh = MockEventHandlerFactory().build(SomeControlHandler()) + assert isinstance(eh, ControlEventHandler) + + eh = MockEventHandlerFactory().build(SomeControlHandler(), event_handler_type=EventHandler) + assert type(eh) == EventHandler # pylint: disable=unidiomatic-typecheck + + SomeControlHandler.type_id = 'FSMAwareEventHandler' + eh = MockEventHandlerFactory().build(SomeControlHandler()) + assert isinstance(eh, FSMAwareEventHandler) + + eh = MockEventHandlerFactory().build(lambda *_: None, keys=['cica']) + assert isinstance(eh, EventHandler) + + def test_build_from_callable(test_keys, test_msg): mock_eh = MockCallable() From 9712ebf1021f1b29f30fa51444dd01720f78d318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 30 Aug 2019 14:45:53 +0200 Subject: [PATCH 152/174] Specify EventHandler types in builtin handlers --- tfw/components/frontend/frontend_proxy_handler.py | 1 + .../frontend/message_queue_handler/message_queue_handler.py | 1 + tfw/components/fsm/fsm_handler.py | 1 + tfw/components/ide/ide_handler.py | 1 + tfw/components/process_management/process_handler.py | 1 + tfw/components/process_management/process_log_handler.py | 1 + tfw/components/terminal/terminal_handler.py | 1 + 7 files changed, 7 insertions(+) diff --git a/tfw/components/frontend/frontend_proxy_handler.py b/tfw/components/frontend/frontend_proxy_handler.py index 6a364d2..d48d8c9 100644 --- a/tfw/components/frontend/frontend_proxy_handler.py +++ b/tfw/components/frontend/frontend_proxy_handler.py @@ -5,6 +5,7 @@ from .message_storage import FrontendMessageStorage class FrontendProxyHandler: keys = ['console', 'dashboard', 'frontend', 'message', 'ide.read', 'deploy.finish'] + type_id = 'ControlEventHandler' def __init__(self): self.connector = None diff --git a/tfw/components/frontend/message_queue_handler/message_queue_handler.py b/tfw/components/frontend/message_queue_handler/message_queue_handler.py index b60dc6a..97f344b 100644 --- a/tfw/components/frontend/message_queue_handler/message_queue_handler.py +++ b/tfw/components/frontend/message_queue_handler/message_queue_handler.py @@ -5,6 +5,7 @@ from threading import Thread class MessageQueueHandler: keys = ['message.queue'] + type_id = 'ControlEventHandler' def __init__(self, wpm): self.connector = None diff --git a/tfw/components/fsm/fsm_handler.py b/tfw/components/fsm/fsm_handler.py index 8f19a3a..2d31594 100644 --- a/tfw/components/fsm/fsm_handler.py +++ b/tfw/components/fsm/fsm_handler.py @@ -10,6 +10,7 @@ LOG = logging.getLogger(__name__) class FSMHandler: keys = ['fsm'] + type_id = 'ControlEventHandler' def __init__(self, *, fsm_type): self.connector = None diff --git a/tfw/components/ide/ide_handler.py b/tfw/components/ide/ide_handler.py index bf1307b..e75ea84 100644 --- a/tfw/components/ide/ide_handler.py +++ b/tfw/components/ide/ide_handler.py @@ -34,6 +34,7 @@ BUILD_ARTIFACTS = ( class IdeHandler: keys = ['ide'] + type_id = 'ControlEventHandler' def __init__(self, *, patterns, initial_file=''): self.connector = None diff --git a/tfw/components/process_management/process_handler.py b/tfw/components/process_management/process_handler.py index 04c121b..c56a02b 100644 --- a/tfw/components/process_management/process_handler.py +++ b/tfw/components/process_management/process_handler.py @@ -10,6 +10,7 @@ LOG = logging.getLogger(__name__) class ProcessHandler(ProcessManager): keys = ['process'] + type_id = 'ControlEventHandler' def __init__(self, *, supervisor_uri): ProcessManager.__init__(self, supervisor_uri) diff --git a/tfw/components/process_management/process_log_handler.py b/tfw/components/process_management/process_log_handler.py index 9e3bb42..a61f806 100644 --- a/tfw/components/process_management/process_log_handler.py +++ b/tfw/components/process_management/process_log_handler.py @@ -7,6 +7,7 @@ LOG = logging.getLogger(__name__) class ProcessLogHandler: keys = ['process.log'] + type_id = 'ControlEventHandler' def __init__(self, *, process_name, supervisor_uri, log_tail=0): self.connector, self._monitor = None, None diff --git a/tfw/components/terminal/terminal_handler.py b/tfw/components/terminal/terminal_handler.py index dd17e97..8044fae 100644 --- a/tfw/components/terminal/terminal_handler.py +++ b/tfw/components/terminal/terminal_handler.py @@ -8,6 +8,7 @@ LOG = logging.getLogger(__name__) class TerminalHandler: keys = ['terminal'] + type_id = 'ControlEventHandler' def __init__(self, *, port, user, working_directory, histfile): self.connector, self._historymonitor = None, None From ffc06a64eec5de9e11039caa117fcc56fe161d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 30 Aug 2019 15:28:11 +0200 Subject: [PATCH 153/174] Clarify exception message in EHFactoryBase --- tfw/internals/event_handling/event_handler_factory_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfw/internals/event_handling/event_handler_factory_base.py b/tfw/internals/event_handling/event_handler_factory_base.py index dd76329..8c5df21 100644 --- a/tfw/internals/event_handling/event_handler_factory_base.py +++ b/tfw/internals/event_handling/event_handler_factory_base.py @@ -26,7 +26,7 @@ class EventHandlerBuilder: def _determine_type(self, provided_type): type_ = provided_type or self._analyzer.event_handler_type or EventHandler if not issubclass(type_, EventHandler): - raise ValueError("No type supplied!") + raise ValueError("Invalid type supplied!") return type_ def build(self, connector): From bb5c000b1d94315bbd9d6c0b69357af52285f32b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 30 Aug 2019 17:37:43 +0200 Subject: [PATCH 154/174] Implement builtin handler for configuring frontend --- tfw/components/frontend/__init__.py | 1 + .../frontend_config_handler/__init__.py | 1 + .../frontend_config_handler.py | 36 +++++++++++++++++ .../test_frontend_config_handler.py | 40 +++++++++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 tfw/components/frontend/frontend_config_handler/__init__.py create mode 100644 tfw/components/frontend/frontend_config_handler/frontend_config_handler.py create mode 100644 tfw/components/frontend/frontend_config_handler/test_frontend_config_handler.py diff --git a/tfw/components/frontend/__init__.py b/tfw/components/frontend/__init__.py index a0bf733..c952158 100644 --- a/tfw/components/frontend/__init__.py +++ b/tfw/components/frontend/__init__.py @@ -3,3 +3,4 @@ from .frontend_proxy_handler import FrontendProxyHandler from .frontend_ready_handler import FrontendReadyHandler from .message_queue_handler import MessageQueueHandler from .message_sender import MessageSender +from .frontend_config_handler import FrontendConfigHandler diff --git a/tfw/components/frontend/frontend_config_handler/__init__.py b/tfw/components/frontend/frontend_config_handler/__init__.py new file mode 100644 index 0000000..86c5b0a --- /dev/null +++ b/tfw/components/frontend/frontend_config_handler/__init__.py @@ -0,0 +1 @@ +from .frontend_config_handler import FrontendConfigHandler diff --git a/tfw/components/frontend/frontend_config_handler/frontend_config_handler.py b/tfw/components/frontend/frontend_config_handler/frontend_config_handler.py new file mode 100644 index 0000000..4c71097 --- /dev/null +++ b/tfw/components/frontend/frontend_config_handler/frontend_config_handler.py @@ -0,0 +1,36 @@ +from yaml import safe_load + +from tfw.internals.networking import Scope + + +class FrontendConfigHandler: + keys = ['frontend.ready'] + + def __init__(self, config_path): + self._config_path = config_path + + def handle_event(self, _, connector): + # pylint: disable=no-self-use + for message in self._config_messages: + connector.send_message(message) + connector.send_message({'key': 'frontend.ready'}, scope=Scope.WEBSOCKET) + + @property + def _config_messages(self): + config = safe_load(self._config_str) + return [ + self._create_config_message(k, v) + for k, v in config.items() + ] + + @property + def _config_str(self): + with open(self._config_path, 'r') as ifile: + return ifile.read() + + @staticmethod + def _create_config_message(key, value): + return { + 'key': f'frontend.{key}', + **value + } diff --git a/tfw/components/frontend/frontend_config_handler/test_frontend_config_handler.py b/tfw/components/frontend/frontend_config_handler/test_frontend_config_handler.py new file mode 100644 index 0000000..007e5f3 --- /dev/null +++ b/tfw/components/frontend/frontend_config_handler/test_frontend_config_handler.py @@ -0,0 +1,40 @@ +from textwrap import dedent + +from .frontend_config_handler import FrontendConfigHandler + + +class MockFrontendConfigHandler(FrontendConfigHandler): + @property + def _config_str(self): + return dedent(''' + cat: + someConfigKey: lel + otherStuff: 42 + foo: + hey: yaaay + ''') + + +class MockConnector: + def __init__(self): + self.messages = [] + + def send_message(self, message, **_): + self.messages.append(message) + + +def test_frontend_config_handler(): + connector = MockConnector() + handler = MockFrontendConfigHandler('') + + handler.handle_event({'key': 'frontend.ready'}, connector) + assert connector.messages[0] == { + 'key': 'frontend.cat', + 'someConfigKey': 'lel', + 'otherStuff': 42 + } + assert connector.messages[1] == { + 'key': 'frontend.foo', + 'hey': 'yaaay' + } + assert connector.messages[-1] == {'key': 'frontend.ready'} From a5db2f23a003b1f5850734fdf33ad56c3943b6f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 30 Aug 2019 17:45:54 +0200 Subject: [PATCH 155/174] Log PipeConnector events --- .../pipe_io/pipe_connector/pipe_connector.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tfw/components/pipe_io/pipe_connector/pipe_connector.py b/tfw/components/pipe_io/pipe_connector/pipe_connector.py index 820ff3f..04157ff 100644 --- a/tfw/components/pipe_io/pipe_connector/pipe_connector.py +++ b/tfw/components/pipe_io/pipe_connector/pipe_connector.py @@ -1,3 +1,4 @@ +import logging from stat import S_ISFIFO from os import access, mkdir, stat, R_OK from os.path import exists @@ -6,6 +7,8 @@ from pipe_io_server import PipeWriterServer from tfw.internals.inotify import InotifyObserver, InotifyFileCreatedEvent, InotifyFileDeletedEvent +LOG = logging.getLogger(__name__) + class PipeConnector: reader_pattern, writer_pattern = 'send', 'recv' @@ -27,15 +30,17 @@ class PipeConnector: path = event.src_path if self._is_pipe(path): if isinstance(event, InotifyFileCreatedEvent): - self._create_pipe(path) + self._on_create_pipe(path) + LOG.debug('Connected to new pipe "%s"', path) elif isinstance(event, InotifyFileDeletedEvent): - self._delete_pipe(path) + self._on_delete_pipe(path) + LOG.debug('Disconnected from deleted pipe "%s"', path) @staticmethod def _is_pipe(path): return exists(path) and S_ISFIFO(stat(path).st_mode) - def _create_pipe(self, path): + def _on_create_pipe(self, path): if self._find_pipe(path): return server = None @@ -61,7 +66,7 @@ class PipeConnector: def build_writer(self, path): # pylint: disable=no-self-use return PipeWriterServer(path, manage_pipes=False) - def _delete_pipe(self, path): + def _on_delete_pipe(self, path): pipes = self._find_pipe(path) if pipes: pipes[path].stop() From 517684e84a50b3d874825b85dc866077cb1cc924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 30 Aug 2019 17:47:58 +0200 Subject: [PATCH 156/174] Fix TerminalCommandsHandler using old JSON API --- tfw/components/terminal/terminal_commands_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfw/components/terminal/terminal_commands_handler.py b/tfw/components/terminal/terminal_commands_handler.py index dcfd5f0..e47aee5 100644 --- a/tfw/components/terminal/terminal_commands_handler.py +++ b/tfw/components/terminal/terminal_commands_handler.py @@ -5,5 +5,5 @@ class TerminalCommandsHandler(TerminalCommands): keys = ['history.bash'] def handle_event(self, message, _): - command = message['value'] + command = message['command'] self.callback(command) From 57ce35d99d347499fe97f1963a8868ae18b50cf1 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 2 Sep 2019 10:46:18 +0200 Subject: [PATCH 157/174] Fix pipe deletion check --- .../pipe_io/pipe_connector/pipe_connector.py | 13 ++++++------- .../pipe_connector/test_proxy_pipe_connector.py | 3 ++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tfw/components/pipe_io/pipe_connector/pipe_connector.py b/tfw/components/pipe_io/pipe_connector/pipe_connector.py index 04157ff..0c33a59 100644 --- a/tfw/components/pipe_io/pipe_connector/pipe_connector.py +++ b/tfw/components/pipe_io/pipe_connector/pipe_connector.py @@ -28,13 +28,12 @@ class PipeConnector: def _on_any_event(self, event): path = event.src_path - if self._is_pipe(path): - if isinstance(event, InotifyFileCreatedEvent): - self._on_create_pipe(path) - LOG.debug('Connected to new pipe "%s"', path) - elif isinstance(event, InotifyFileDeletedEvent): - self._on_delete_pipe(path) - LOG.debug('Disconnected from deleted pipe "%s"', path) + if self._is_pipe(path) and isinstance(event, InotifyFileCreatedEvent): + self._on_create_pipe(path) + LOG.debug('Connected to new pipe "%s"', path) + elif isinstance(event, InotifyFileDeletedEvent): + self._on_delete_pipe(path) + LOG.debug('Disconnected from deleted pipe "%s"', path) @staticmethod def _is_pipe(path): diff --git a/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py b/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py index 0764d0c..65175f7 100644 --- a/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py +++ b/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py @@ -3,7 +3,7 @@ from enum import Enum from dataclasses import dataclass from json import dumps from secrets import token_urlsafe -from os import urandom, mkfifo, mkdir +from os import urandom, mkfifo, mkdir, remove from os.path import join from pathlib import Path from tempfile import TemporaryDirectory @@ -117,6 +117,7 @@ def test_pipe_creation_deletion(mock_context): path = mock_context.emit_pipe_creation_event(action, mkfifo) assert events[-1] == path assert path in pipes.keys() + remove(path) mock_context.pipes.observer.on_any_event(InotifyFileDeletedEvent(path)) assert path not in pipes.keys() From 1d8de09c2aaabb163225ec25501f7f5c944d734c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 2 Sep 2019 15:49:13 +0200 Subject: [PATCH 158/174] Make Filemanager.is_allowed() public --- tfw/components/ide/file_manager/file_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tfw/components/ide/file_manager/file_manager.py b/tfw/components/ide/file_manager/file_manager.py index e416cb9..994e57a 100644 --- a/tfw/components/ide/file_manager/file_manager.py +++ b/tfw/components/ide/file_manager/file_manager.py @@ -7,7 +7,7 @@ from os.path import dirname, isdir, isfile, realpath def _with_is_allowed(func): @wraps(func) def wrapper(self, *args, **kwargs): - if self._is_allowed(args[0]): # pylint: disable=protected-access + if self.is_allowed(args[0]): # pylint: disable=protected-access return func(self, *args, **kwargs) raise ValueError('Forbidden path.') return wrapper @@ -23,7 +23,7 @@ class FileManager: # pylint: disable=too-many-instance-attributes path for pattern in self.patterns for path in glob(pattern, recursive=True) - if isfile(path) and self._is_allowed(path) + if isfile(path) and self.is_allowed(path) )) @property @@ -39,7 +39,7 @@ class FileManager: # pylint: disable=too-many-instance-attributes pattern = dirname(pattern) return pattern - def _is_allowed(self, filepath): + def is_allowed(self, filepath): return any( fnmatchcase(realpath(filepath), pattern) for pattern in self.patterns From 701b918e6ff48ec3c0325216208d28a21f974390 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 2 Sep 2019 15:49:38 +0200 Subject: [PATCH 159/174] Refactor IdeHandler --- tfw/components/ide/ide_handler.py | 61 ++++++++++++++++--------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/tfw/components/ide/ide_handler.py b/tfw/components/ide/ide_handler.py index e75ea84..d4ef8a1 100644 --- a/tfw/components/ide/ide_handler.py +++ b/tfw/components/ide/ide_handler.py @@ -1,5 +1,4 @@ import logging -from os.path import isfile from tfw.internals.networking import Scope from tfw.internals.inotify import InotifyObserver @@ -36,10 +35,10 @@ class IdeHandler: keys = ['ide'] type_id = 'ControlEventHandler' - def __init__(self, *, patterns, initial_file=''): + def __init__(self, *, patterns, initial_file=None): self.connector = None self.filemanager = FileManager(patterns) - self._initial_file = initial_file + self._initial_file = initial_file or '' self.monitor = InotifyObserver( path=self.filemanager.parents, @@ -53,46 +52,48 @@ class IdeHandler: 'ide.write' : self.write } - @property - def initial_file(self): - if not isfile(self._initial_file): - self._initial_file = self.filemanager.files[0] - return self._initial_file - def _reload_frontend(self, event): # pylint: disable=unused-argument self.send_message({'key': 'ide.reload'}) + @property + def initial_file(self): + if not self.filemanager.is_allowed(self._initial_file): + self._initial_file = self.filemanager.files[0] + return self._initial_file + def send_message(self, message): self.connector.send_message(message, scope=Scope.WEBSOCKET) - def read(self, message): - if message.get('files'): - self.filemanager.patterns = message['files'] - try: - message['content'] = self.filemanager.read_file(message['filename']) - except PermissionError: - message['content'] = 'You have no permission to open that file :(' - except FileNotFoundError: - message['content'] = 'This file was removed :(' - except Exception: # pylint: disable=broad-except - message['content'] = 'Failed to read file :(' - - def write(self, message): - try: - self.filemanager.write_file(message['filename'], message['content']) - except Exception: # pylint: disable=broad-except - LOG.exception('Error writing file!') - del message['content'] - def handle_event(self, message, _): try: - if message['filename'] == '': - message['filename'] = self.initial_file self.commands[message['key']](message) message['files'] = self.filemanager.files self.send_message(message) except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) + def read(self, message): + if message.get('files'): + self.filemanager.patterns = message['files'] + try: + message['filename'] = message.get('filename') or self.initial_file + message['content'] = self.filemanager.read_file(message['filename']) + except (PermissionError, ValueError): + message['content'] = 'You have no permission to open that file :(' + except FileNotFoundError: + message['content'] = 'This file does not exist :(' + except Exception: # pylint: disable=broad-except + message['content'] = 'Failed to read file :(' + LOG.exception('Error reading file!') + + def write(self, message): + try: + self.filemanager.write_file(message['filename'], message['content']) + except KeyError: + LOG.error('You must provide a filename to write!') + except Exception: # pylint: disable=broad-except + LOG.exception('Error writing file!') + del message['content'] + def cleanup(self): self.monitor.stop() From fab582505f640b0d263f3379872483ddc4b005f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 2 Sep 2019 16:21:19 +0200 Subject: [PATCH 160/174] Improve IdeHandler JSON API --- tfw/components/ide/ide_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tfw/components/ide/ide_handler.py b/tfw/components/ide/ide_handler.py index d4ef8a1..f0157dc 100644 --- a/tfw/components/ide/ide_handler.py +++ b/tfw/components/ide/ide_handler.py @@ -73,8 +73,8 @@ class IdeHandler: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) def read(self, message): - if message.get('files'): - self.filemanager.patterns = message['files'] + if 'patterns' in message: + self.filemanager.patterns = message['patterns'] try: message['filename'] = message.get('filename') or self.initial_file message['content'] = self.filemanager.read_file(message['filename']) From 777dc9ccfcd621a25882b013d6fd1aa7f0ca8baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 2 Sep 2019 16:25:04 +0200 Subject: [PATCH 161/174] Remove unnecessary pylint: disable comment --- tfw/components/ide/file_manager/file_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfw/components/ide/file_manager/file_manager.py b/tfw/components/ide/file_manager/file_manager.py index 994e57a..be74e22 100644 --- a/tfw/components/ide/file_manager/file_manager.py +++ b/tfw/components/ide/file_manager/file_manager.py @@ -7,7 +7,7 @@ from os.path import dirname, isdir, isfile, realpath def _with_is_allowed(func): @wraps(func) def wrapper(self, *args, **kwargs): - if self.is_allowed(args[0]): # pylint: disable=protected-access + if self.is_allowed(args[0]): return func(self, *args, **kwargs) raise ValueError('Forbidden path.') return wrapper From 66c9c5a592de181ffba3cb261ff4601f58e4942d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 4 Sep 2019 15:13:11 +0200 Subject: [PATCH 162/174] Implement finding files from relative paths FileManager --- tfw/components/ide/file_manager/file_manager.py | 12 +++++++++++- tfw/components/ide/file_manager/test_file_manager.py | 8 +++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/tfw/components/ide/file_manager/file_manager.py b/tfw/components/ide/file_manager/file_manager.py index be74e22..1ebc5e0 100644 --- a/tfw/components/ide/file_manager/file_manager.py +++ b/tfw/components/ide/file_manager/file_manager.py @@ -1,7 +1,10 @@ +import logging from functools import wraps from glob import glob from fnmatch import fnmatchcase -from os.path import dirname, isdir, isfile, realpath +from os.path import dirname, isdir, isfile, realpath, isabs + +LOG = logging.getLogger(__name__) def _with_is_allowed(func): @@ -45,6 +48,13 @@ class FileManager: # pylint: disable=too-many-instance-attributes for pattern in self.patterns ) + def find_file(self, filename): + if not isabs(filename): + for filepath in self.files: + if filepath.endswith(filename): + return filepath + return filename + @_with_is_allowed def read_file(self, filepath): # pylint: disable=no-self-use with open(filepath, 'rb', buffering=0) as ifile: diff --git a/tfw/components/ide/file_manager/test_file_manager.py b/tfw/components/ide/file_manager/test_file_manager.py index d321ea3..b634b0d 100644 --- a/tfw/components/ide/file_manager/test_file_manager.py +++ b/tfw/components/ide/file_manager/test_file_manager.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from secrets import token_urlsafe from os import mkdir, symlink -from os.path import join, realpath +from os.path import join, realpath, basename from pathlib import Path from tempfile import TemporaryDirectory @@ -86,3 +86,9 @@ def test_regular_ide_actions(context): context.manager.write_file(newfile2, content2) assert context.manager.read_file(newfile1) == content1 assert context.manager.read_file(newfile2) == content2 + +def test_find_file(context): + for _ in range(5): + file_abs = context.create_random_file(context.subdir, '.txt') + file_base = basename(file_abs) + assert context.manager.find_file(file_base) == file_abs From 620212f00f065a2f6286ff98247e2284de5e5453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 4 Sep 2019 15:13:40 +0200 Subject: [PATCH 163/174] Accept relative paths in the IdeHandler API's "filename" key --- tfw/components/ide/ide_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tfw/components/ide/ide_handler.py b/tfw/components/ide/ide_handler.py index f0157dc..3a86da4 100644 --- a/tfw/components/ide/ide_handler.py +++ b/tfw/components/ide/ide_handler.py @@ -76,7 +76,9 @@ class IdeHandler: if 'patterns' in message: self.filemanager.patterns = message['patterns'] try: - message['filename'] = message.get('filename') or self.initial_file + message['filename'] = self.filemanager.find_file( + message.get('filename') or self.initial_file + ) message['content'] = self.filemanager.read_file(message['filename']) except (PermissionError, ValueError): message['content'] = 'You have no permission to open that file :(' From 10cd07973dfe8fc225c0cfcd64dfe9fd5ebc67fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 4 Sep 2019 17:13:58 +0200 Subject: [PATCH 164/174] Implement unit tests for EventHandler subclasses --- .../test_event_handling/__init__.py | 0 .../test_control_event_handler.py | 33 +++++++++ .../test_event_handler.py | 8 +-- .../test_fsm_aware_event_handler.py | 69 +++++++++++++++++++ .../test_signed_event_handler.py | 32 +++++++++ .../test_event_handling/util.py | 11 +++ 6 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 tfw/internals/event_handling/test_event_handling/__init__.py create mode 100644 tfw/internals/event_handling/test_event_handling/test_control_event_handler.py rename tfw/internals/event_handling/{ => test_event_handling}/test_event_handler.py (96%) create mode 100644 tfw/internals/event_handling/test_event_handling/test_fsm_aware_event_handler.py create mode 100644 tfw/internals/event_handling/test_event_handling/test_signed_event_handler.py create mode 100644 tfw/internals/event_handling/test_event_handling/util.py diff --git a/tfw/internals/event_handling/test_event_handling/__init__.py b/tfw/internals/event_handling/test_event_handling/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tfw/internals/event_handling/test_event_handling/test_control_event_handler.py b/tfw/internals/event_handling/test_event_handling/test_control_event_handler.py new file mode 100644 index 0000000..057a9c5 --- /dev/null +++ b/tfw/internals/event_handling/test_event_handling/test_control_event_handler.py @@ -0,0 +1,33 @@ +from tfw.internals.networking import Intent + +from .util import DummyConnector, simulate_event +from ..control_event_handler import ControlEventHandler + + +def test_receives_control_intent(): + eh = ControlEventHandler(DummyConnector()) + messages = [] + eh.handle_event = lambda msg, _: messages.append(msg) + + test_msg = {"key": "cica", "intent": Intent.CONTROL.value} + simulate_event(eh, test_msg) + assert messages[0] == test_msg + + +def test_no_intent_defaults_to_control(): + eh = ControlEventHandler(DummyConnector()) + messages = [] + eh.handle_event = lambda msg, _: messages.append(msg) + + test_msg = {"key": "cica"} + simulate_event(eh, test_msg) + assert messages[0] == test_msg + + +def test_ignores_event_intent(): + eh = ControlEventHandler(DummyConnector()) + messages = [] + eh.handle_event = lambda msg, _: messages.append(msg) + + simulate_event(eh, {"key": "cica", "intent": Intent.EVENT.value}) + assert not messages diff --git a/tfw/internals/event_handling/test_event_handler.py b/tfw/internals/event_handling/test_event_handling/test_event_handler.py similarity index 96% rename from tfw/internals/event_handling/test_event_handler.py rename to tfw/internals/event_handling/test_event_handling/test_event_handler.py index 5082ca8..e206b46 100644 --- a/tfw/internals/event_handling/test_event_handler.py +++ b/tfw/internals/event_handling/test_event_handling/test_event_handler.py @@ -4,10 +4,10 @@ from random import randint import pytest -from .event_handler_factory_base import EventHandlerFactoryBase -from .event_handler import EventHandler -from .control_event_handler import ControlEventHandler -from .fsm_aware_event_handler import FSMAwareEventHandler +from ..event_handler_factory_base import EventHandlerFactoryBase +from ..event_handler import EventHandler +from ..control_event_handler import ControlEventHandler +from ..fsm_aware_event_handler import FSMAwareEventHandler class MockEventHandlerFactory(EventHandlerFactoryBase): diff --git a/tfw/internals/event_handling/test_event_handling/test_fsm_aware_event_handler.py b/tfw/internals/event_handling/test_event_handling/test_fsm_aware_event_handler.py new file mode 100644 index 0000000..2c37874 --- /dev/null +++ b/tfw/internals/event_handling/test_event_handling/test_fsm_aware_event_handler.py @@ -0,0 +1,69 @@ +# pylint: disable=redefined-outer-name +import pytest + +from tfw.internals.networking import Intent +from tfw.internals.crypto import KeyManager, sign_message + +from .util import DummyConnector, simulate_event +from ..fsm_aware_event_handler import FSMAwareEventHandler + + +@pytest.fixture +def key(): + yield KeyManager().auth_key + + +@pytest.fixture +def fsm_update_msg(): + yield { + 'key': 'fsm.update', + 'intent': Intent.EVENT.value, + 'scope': 'broadcast', + 'current_state': '1', + 'in_accepted_state': False, + 'last_event': { + 'from_state': '0', + 'timestamp': '2019-09-04T16:51:15.587555', + 'to_state': '1', 'trigger': + 'step_1' + }, + 'valid_transitions': [{'trigger': 'step_2'}] + } + + +def test_ignores_other_keys(): + messages = [] + eh = FSMAwareEventHandler(DummyConnector()) + eh.handle_event = lambda msg, _: messages.append(msg) + + simulate_event(eh, {"key": "not.fsm.update"}) + assert not messages + + +def test_ignores_unauthenticated(fsm_update_msg): + messages = [] + eh = FSMAwareEventHandler(DummyConnector()) + eh.handle_event = lambda msg, _: messages.append(msg) + + simulate_event(eh, fsm_update_msg) + assert not messages + + +def test_accepts_authenticated(key, fsm_update_msg): + messages = [] + eh = FSMAwareEventHandler(DummyConnector()) + eh.handle_event = lambda msg, _: messages.append(msg) + + sign_message(key, fsm_update_msg) + simulate_event(eh, fsm_update_msg) + assert messages[0] == fsm_update_msg + assert eh.fsm_state == '1' + assert not eh.fsm_in_accepted_state + assert len(eh.fsm_event_log) == 1 + + fsm_update_msg['in_accepted_state'] = True + fsm_update_msg['current_state'] = '2' + sign_message(key, fsm_update_msg) + simulate_event(eh, fsm_update_msg) + assert eh.fsm_state == '2' + assert eh.fsm_in_accepted_state diff --git a/tfw/internals/event_handling/test_event_handling/test_signed_event_handler.py b/tfw/internals/event_handling/test_event_handling/test_signed_event_handler.py new file mode 100644 index 0000000..fbf7e43 --- /dev/null +++ b/tfw/internals/event_handling/test_event_handling/test_signed_event_handler.py @@ -0,0 +1,32 @@ +# pylint: disable=redefined-outer-name +import pytest + +from tfw.internals.crypto import sign_message, KeyManager + +from .util import DummyConnector, simulate_event +from ..signed_event_handler import SignedEventHandler + + +@pytest.fixture +def key(): + yield KeyManager().auth_key + + +def test_ignores_unauthenticated(): + eh = SignedEventHandler(DummyConnector()) + messages = [] + eh.handle_event = lambda msg, _: messages.append(msg) + + simulate_event(eh, {"key": "cica"}) + assert not messages + + +def test_accepts_authenticated(key): + eh = SignedEventHandler(DummyConnector()) + messages = [] + eh.handle_event = lambda msg, _: messages.append(msg) + + test_msg = {"key": "cica"} + sign_message(key, test_msg) + simulate_event(eh, test_msg) + assert messages[0] == test_msg diff --git a/tfw/internals/event_handling/test_event_handling/util.py b/tfw/internals/event_handling/test_event_handling/util.py new file mode 100644 index 0000000..1dbd171 --- /dev/null +++ b/tfw/internals/event_handling/test_event_handling/util.py @@ -0,0 +1,11 @@ +class DummyConnector(): + def send_message(self, message, scope=None, intent=None): + pass + + def close(self): + pass + + +def simulate_event(event_handler, message): + # pylint: disable=protected-access + event_handler._event_callback(message) From b733f53b839a122bab36351cebad9f1cc64a03ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 4 Sep 2019 17:21:49 +0200 Subject: [PATCH 165/174] Remove unused but beautiful product of me being with my muse :( --- tfw/components/pipe_io/__init__.py | 2 +- tfw/components/pipe_io/pipe_io_handler.py | 37 ----------------------- 2 files changed, 1 insertion(+), 38 deletions(-) diff --git a/tfw/components/pipe_io/__init__.py b/tfw/components/pipe_io/__init__.py index 54bc368..2160960 100644 --- a/tfw/components/pipe_io/__init__.py +++ b/tfw/components/pipe_io/__init__.py @@ -1,2 +1,2 @@ -from .pipe_io_handler import PipeIOHandler, PipeIOHandlerBase, TransformerPipeIOHandler, CommandHandler +from .pipe_io_handler import PipeIOHandler, PipeIOHandlerBase, CommandHandler from .pipe_connector import ProxyPipeConnectorHandler diff --git a/tfw/components/pipe_io/pipe_io_handler.py b/tfw/components/pipe_io/pipe_io_handler.py index 671f1db..55ea133 100644 --- a/tfw/components/pipe_io/pipe_io_handler.py +++ b/tfw/components/pipe_io/pipe_io_handler.py @@ -59,43 +59,6 @@ class PipeIOHandler(PipeIOHandlerBase): self.connector.send_message(json) -class TransformerPipeIOHandler(PipeIOHandlerBase): - # pylint: disable=too-many-arguments - def __init__( - self, in_pipe_path, out_pipe_path, - transform_in_cmd, transform_out_cmd, - permissions=DEFAULT_PERMISSIONS - ): - self._transform_in = partial(self._transform_message, transform_in_cmd) - self._transform_out = partial(self._transform_message, transform_out_cmd) - super().__init__(in_pipe_path, out_pipe_path, permissions) - - @staticmethod - def _transform_message(transform_cmd, message): - proc = run( - transform_cmd, - input=message, - stdout=PIPE, - stderr=PIPE, - shell=True - ) - if proc.returncode == 0: - return proc.stdout - raise ValueError(f'Transforming message {message} failed!') - - def handle_event(self, message, _): - json_bytes = dumps(message).encode() - transformed_bytes = self._transform_out(json_bytes) - if transformed_bytes: - self.pipe_io.send_message(transformed_bytes) - - def handle_pipe_event(self, message_bytes): - transformed_bytes = self._transform_in(message_bytes) - if transformed_bytes: - json_message = loads(transformed_bytes) - self.connector.send_message(json_message) - - class CommandHandler(PipeIOHandler): def __init__(self, command, permissions=DEFAULT_PERMISSIONS): super().__init__( From 613d906b4478d7e886dd6d4e1adc5bb838a4d916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 4 Sep 2019 18:07:36 +0200 Subject: [PATCH 166/174] Simplify PipeIOHandlerBase by using meghod assignment --- tfw/components/pipe_io/pipe_io_handler.py | 25 +++++++++-------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tfw/components/pipe_io/pipe_io_handler.py b/tfw/components/pipe_io/pipe_io_handler.py index 55ea133..d5726a1 100644 --- a/tfw/components/pipe_io/pipe_io_handler.py +++ b/tfw/components/pipe_io/pipe_io_handler.py @@ -1,7 +1,7 @@ import logging from abc import abstractmethod from json import loads, dumps -from subprocess import run, PIPE, Popen +from subprocess import PIPE, Popen from functools import partial from os import getpgid, killpg from os.path import join @@ -21,14 +21,21 @@ class PipeIOHandlerBase: def __init__(self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS): self.connector = None - self.pipe_io = CallbackPipeIOServer( + self._in_pipe = in_pipe_path + self.pipe_io = PipeIOServer( in_pipe_path, out_pipe_path, - self.handle_pipe_event, permissions ) + self.pipe_io.handle_message = self._server_handle_message self.pipe_io.start() + def _server_handle_message(self, message): + try: + self.handle_pipe_event(message) + except: # pylint: disable=bare-except + LOG.exception('Failed to handle message %s from pipe %s!', message, self._in_pipe) + @abstractmethod def handle_pipe_event(self, message_bytes): raise NotImplementedError() @@ -37,18 +44,6 @@ class PipeIOHandlerBase: self.pipe_io.stop() -class CallbackPipeIOServer(PipeIOServer): - def __init__(self, in_pipe_path, out_pipe_path, callback, permissions): - super().__init__(in_pipe_path, out_pipe_path, permissions) - self.callback = callback - - def handle_message(self, message): - try: - self.callback(message) - except: # pylint: disable=bare-except - LOG.exception('Failed to handle message %s from pipe %s!', message, self.in_pipe) - - class PipeIOHandler(PipeIOHandlerBase): def handle_event(self, message, _): json_bytes = dumps(message).encode() From 89b720c439b17b56a49566801bd94c99e707cecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 4 Sep 2019 18:18:03 +0200 Subject: [PATCH 167/174] Add forgotten test case --- .../test_fsm_aware_event_handler.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tfw/internals/event_handling/test_event_handling/test_fsm_aware_event_handler.py b/tfw/internals/event_handling/test_event_handling/test_fsm_aware_event_handler.py index 2c37874..c08b8d0 100644 --- a/tfw/internals/event_handling/test_event_handling/test_fsm_aware_event_handler.py +++ b/tfw/internals/event_handling/test_event_handling/test_fsm_aware_event_handler.py @@ -31,15 +31,6 @@ def fsm_update_msg(): } -def test_ignores_other_keys(): - messages = [] - eh = FSMAwareEventHandler(DummyConnector()) - eh.handle_event = lambda msg, _: messages.append(msg) - - simulate_event(eh, {"key": "not.fsm.update"}) - assert not messages - - def test_ignores_unauthenticated(fsm_update_msg): messages = [] eh = FSMAwareEventHandler(DummyConnector()) @@ -49,6 +40,17 @@ def test_ignores_unauthenticated(fsm_update_msg): assert not messages +def test_ignores_other_keys(key): + messages = [] + eh = FSMAwareEventHandler(DummyConnector()) + eh.handle_event = lambda msg, _: messages.append(msg) + + test_msg = {"key": "not.fsm.update"} + sign_message(key, test_msg) + simulate_event(eh, test_msg) + assert not messages + + def test_accepts_authenticated(key, fsm_update_msg): messages = [] eh = FSMAwareEventHandler(DummyConnector()) From 8890e67b86d876b8e100c87589bc613f0c06d93c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 5 Sep 2019 15:52:34 +0200 Subject: [PATCH 168/174] Implement test cases for PipeIOHandler --- tfw/components/pipe_io/pipe_io_handler.py | 5 ++- .../pipe_io/test_pipe_io_handler.py | 45 +++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 tfw/components/pipe_io/test_pipe_io_handler.py diff --git a/tfw/components/pipe_io/pipe_io_handler.py b/tfw/components/pipe_io/pipe_io_handler.py index d5726a1..50201ac 100644 --- a/tfw/components/pipe_io/pipe_io_handler.py +++ b/tfw/components/pipe_io/pipe_io_handler.py @@ -21,7 +21,8 @@ class PipeIOHandlerBase: def __init__(self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS): self.connector = None - self._in_pipe = in_pipe_path + self.in_pipe = in_pipe_path + self.out_pipe = out_pipe_path self.pipe_io = PipeIOServer( in_pipe_path, out_pipe_path, @@ -34,7 +35,7 @@ class PipeIOHandlerBase: try: self.handle_pipe_event(message) except: # pylint: disable=bare-except - LOG.exception('Failed to handle message %s from pipe %s!', message, self._in_pipe) + LOG.exception('Failed to handle message %s from pipe %s!', message, self.in_pipe) @abstractmethod def handle_pipe_event(self, message_bytes): diff --git a/tfw/components/pipe_io/test_pipe_io_handler.py b/tfw/components/pipe_io/test_pipe_io_handler.py new file mode 100644 index 0000000..b4a0aaa --- /dev/null +++ b/tfw/components/pipe_io/test_pipe_io_handler.py @@ -0,0 +1,45 @@ +# pylint: disable=redefined-outer-name +from tempfile import TemporaryDirectory +from os.path import join +from queue import Queue +from json import dumps, loads + +import pytest + +from .pipe_io_handler import PipeIOHandler + + +@pytest.fixture +def pipe_io(): + with TemporaryDirectory() as tmpdir: + pipeio = PipeIOHandler( + join(tmpdir, 'in'), + join(tmpdir, 'out'), + permissions=0o600 + ) + pipeio.connector = MockConnector() + yield pipeio + pipeio.cleanup() + + +class MockConnector: + def __init__(self): + self.messages = Queue() + + def send_message(self, msg): + self.messages.put(msg) + + +def test_pipe_io_handler_recv(pipe_io): + test_msg = {'key': 'cica'} + with open(pipe_io.in_pipe, 'w') as ofile: + ofile.write(dumps(test_msg) + '\n') + + assert pipe_io.connector.messages.get() == test_msg + + +def test_pipe_io_handler_send(pipe_io): + test_msg = {'key': 'cica'} + pipe_io.handle_event(test_msg, None) + with open(pipe_io.out_pipe, 'r') as ifile: + assert loads(ifile.readline()) == test_msg From 5b0f79dbaeba33a7b5ac988c29f26f3a1c278af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 5 Sep 2019 16:10:46 +0200 Subject: [PATCH 169/174] Move CommandHandler to a separate file --- tfw/components/pipe_io/__init__.py | 3 +- tfw/components/pipe_io/command_handler.py | 58 +++++++++++++++++++++++ tfw/components/pipe_io/pipe_io_handler.py | 56 +--------------------- 3 files changed, 61 insertions(+), 56 deletions(-) create mode 100644 tfw/components/pipe_io/command_handler.py diff --git a/tfw/components/pipe_io/__init__.py b/tfw/components/pipe_io/__init__.py index 2160960..6fc4fc0 100644 --- a/tfw/components/pipe_io/__init__.py +++ b/tfw/components/pipe_io/__init__.py @@ -1,2 +1,3 @@ -from .pipe_io_handler import PipeIOHandler, PipeIOHandlerBase, CommandHandler +from .pipe_io_handler import PipeIOHandler, PipeIOHandlerBase +from .command_handler import CommandHandler from .pipe_connector import ProxyPipeConnectorHandler diff --git a/tfw/components/pipe_io/command_handler.py b/tfw/components/pipe_io/command_handler.py new file mode 100644 index 0000000..e8d1593 --- /dev/null +++ b/tfw/components/pipe_io/command_handler.py @@ -0,0 +1,58 @@ +from subprocess import PIPE, Popen +from functools import partial +from os import getpgid, killpg +from os.path import join +from signal import SIGTERM +from secrets import token_urlsafe +from threading import Thread +from contextlib import suppress + +from pipe_io_server import terminate_process_on_failure + +from .pipe_io_handler import PipeIOHandler, DEFAULT_PERMISSIONS + + +class CommandHandler(PipeIOHandler): + def __init__(self, command, permissions=DEFAULT_PERMISSIONS): + super().__init__( + self._generate_tempfilename(), + self._generate_tempfilename(), + permissions + ) + + self._proc_stdin = open(self.pipe_io.out_pipe, 'rb') + self._proc_stdout = open(self.pipe_io.in_pipe, 'wb') + self._proc = Popen( + command, shell=True, executable='/bin/bash', + stdin=self._proc_stdin, stdout=self._proc_stdout, stderr=PIPE, + start_new_session=True + ) + self._monitor_proc_thread = self._start_monitor_proc() + + def _generate_tempfilename(self): + # pylint: disable=no-self-use + random_filename = partial(token_urlsafe, 10) + return join('/tmp', f'{type(self).__name__}.{random_filename()}') + + def _start_monitor_proc(self): + thread = Thread(target=self._monitor_proc, daemon=True) + thread.start() + return thread + + @terminate_process_on_failure + def _monitor_proc(self): + return_code = self._proc.wait() + if return_code == -int(SIGTERM): + # supervisord asked the program to terminate, this is fine + return + if return_code != 0: + _, stderr = self._proc.communicate() + raise RuntimeError(f'Subprocess failed ({return_code})! Stderr:\n{stderr.decode()}') + + def cleanup(self): + with suppress(ProcessLookupError): + process_group_id = getpgid(self._proc.pid) + killpg(process_group_id, SIGTERM) + self._proc_stdin.close() + self._proc_stdout.close() + super().cleanup() diff --git a/tfw/components/pipe_io/pipe_io_handler.py b/tfw/components/pipe_io/pipe_io_handler.py index 50201ac..bea846a 100644 --- a/tfw/components/pipe_io/pipe_io_handler.py +++ b/tfw/components/pipe_io/pipe_io_handler.py @@ -1,16 +1,8 @@ import logging from abc import abstractmethod from json import loads, dumps -from subprocess import PIPE, Popen -from functools import partial -from os import getpgid, killpg -from os.path import join -from signal import SIGTERM -from secrets import token_urlsafe -from threading import Thread -from contextlib import suppress -from pipe_io_server import PipeIOServer, terminate_process_on_failure +from pipe_io_server import PipeIOServer LOG = logging.getLogger(__name__) DEFAULT_PERMISSIONS = 0o600 @@ -53,49 +45,3 @@ class PipeIOHandler(PipeIOHandlerBase): def handle_pipe_event(self, message_bytes): json = loads(message_bytes) self.connector.send_message(json) - - -class CommandHandler(PipeIOHandler): - def __init__(self, command, permissions=DEFAULT_PERMISSIONS): - super().__init__( - self._generate_tempfilename(), - self._generate_tempfilename(), - permissions - ) - - self._proc_stdin = open(self.pipe_io.out_pipe, 'rb') - self._proc_stdout = open(self.pipe_io.in_pipe, 'wb') - self._proc = Popen( - command, shell=True, executable='/bin/bash', - stdin=self._proc_stdin, stdout=self._proc_stdout, stderr=PIPE, - start_new_session=True - ) - self._monitor_proc_thread = self._start_monitor_proc() - - def _generate_tempfilename(self): - # pylint: disable=no-self-use - random_filename = partial(token_urlsafe, 10) - return join('/tmp', f'{type(self).__name__}.{random_filename()}') - - def _start_monitor_proc(self): - thread = Thread(target=self._monitor_proc, daemon=True) - thread.start() - return thread - - @terminate_process_on_failure - def _monitor_proc(self): - return_code = self._proc.wait() - if return_code == -int(SIGTERM): - # supervisord asked the program to terminate, this is fine - return - if return_code != 0: - _, stderr = self._proc.communicate() - raise RuntimeError(f'Subprocess failed ({return_code})! Stderr:\n{stderr.decode()}') - - def cleanup(self): - with suppress(ProcessLookupError): - process_group_id = getpgid(self._proc.pid) - killpg(process_group_id, SIGTERM) - self._proc_stdin.close() - self._proc_stdout.close() - super().cleanup() From d6f2eb987f0f3145b399076c0e410ae4b74a73b3 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 10 Sep 2019 09:54:04 +0200 Subject: [PATCH 170/174] Make PipeConnector stop gracefully and connect to every pipe on init --- .../pipe_io/pipe_connector/pipe_connector.py | 23 +++++-- .../proxy_pipe_connector_handler.py | 3 + .../test_proxy_pipe_connector.py | 60 +++++++++---------- 3 files changed, 48 insertions(+), 38 deletions(-) diff --git a/tfw/components/pipe_io/pipe_connector/pipe_connector.py b/tfw/components/pipe_io/pipe_connector/pipe_connector.py index 0c33a59..fa423da 100644 --- a/tfw/components/pipe_io/pipe_connector/pipe_connector.py +++ b/tfw/components/pipe_io/pipe_connector/pipe_connector.py @@ -1,7 +1,7 @@ import logging from stat import S_ISFIFO -from os import access, mkdir, stat, R_OK -from os.path import exists +from os import access, listdir, mkdir, stat, R_OK +from os.path import exists, join from pipe_io_server import PipeWriterServer @@ -18,6 +18,11 @@ class PipeConnector: self.observer = self.build_observer(path) self.observer.on_any_event = self._on_any_event self.observer.start() + for node in listdir(path): + node_path = join(path, node) + if self._is_pipe(node_path): + self._create_pipe(node_path) + LOG.debug('Connected to existing pipe "%s"', node_path) def build_observer(self, path): # pylint: disable=no-self-use if not exists(path): @@ -29,17 +34,17 @@ class PipeConnector: def _on_any_event(self, event): path = event.src_path if self._is_pipe(path) and isinstance(event, InotifyFileCreatedEvent): - self._on_create_pipe(path) + self._create_pipe(path) LOG.debug('Connected to new pipe "%s"', path) elif isinstance(event, InotifyFileDeletedEvent): - self._on_delete_pipe(path) + self._delete_pipe(path) LOG.debug('Disconnected from deleted pipe "%s"', path) @staticmethod def _is_pipe(path): return exists(path) and S_ISFIFO(stat(path).st_mode) - def _on_create_pipe(self, path): + def _create_pipe(self, path): if self._find_pipe(path): return server = None @@ -65,7 +70,7 @@ class PipeConnector: def build_writer(self, path): # pylint: disable=no-self-use return PipeWriterServer(path, manage_pipes=False) - def _on_delete_pipe(self, path): + def _delete_pipe(self, path): pipes = self._find_pipe(path) if pipes: pipes[path].stop() @@ -74,3 +79,9 @@ class PipeConnector: def broadcast(self, message): for server in self.recv_pipes.values(): server.send_message(message) + + def stop(self): + for pipe in self.recv_pipes.values(): + pipe.stop() + for pipe in self.send_pipes.values(): + pipe.stop() diff --git a/tfw/components/pipe_io/pipe_connector/proxy_pipe_connector_handler.py b/tfw/components/pipe_io/pipe_connector/proxy_pipe_connector_handler.py index b3b20f7..c82ebca 100644 --- a/tfw/components/pipe_io/pipe_connector/proxy_pipe_connector_handler.py +++ b/tfw/components/pipe_io/pipe_connector/proxy_pipe_connector_handler.py @@ -22,6 +22,9 @@ class ProxyPipeConnectorHandler: def handle_event(self, message, _): self.pipes.broadcast(dumps(message).encode()) + def cleanup(self): + self.pipes.stop() + class ProxyPipeConnector(PipeConnector): def __init__(self, path, connector): diff --git a/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py b/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py index 65175f7..e49f7cc 100644 --- a/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py +++ b/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py @@ -11,7 +11,6 @@ from tempfile import TemporaryDirectory import pytest from tfw.internals.inotify import InotifyFileCreatedEvent, InotifyFileDeletedEvent -from .pipe_connector import PipeConnector from .proxy_pipe_connector_handler import ProxyPipeConnector @@ -99,63 +98,60 @@ def workdir(): @pytest.fixture def context(workdir): - yield PipeContext(workdir, PipeConnector(workdir)) - - -@pytest.fixture -def mock_context(workdir): + mkfifo(join(workdir, Action.SEND.value)) + mkfifo(join(workdir, Action.RECV.value)) yield PipeContext(workdir, MockPipeConnector(workdir, MockConnector())) -def test_pipe_creation_deletion(mock_context): +def test_existing_pipe_connection(context): + assert join(context.workdir, Action.SEND.value) in context.pipes.send_pipes.keys() + assert join(context.workdir, Action.RECV.value) in context.pipes.recv_pipes.keys() + + +def test_pipe_creation_deletion(context): cases = [ - (Action.RECV, mock_context.pipes.recv_pipes, mock_context.pipes.writer_events), - (Action.SEND, mock_context.pipes.send_pipes, mock_context.pipes.reader_events) + (Action.RECV, context.pipes.recv_pipes, context.pipes.writer_events), + (Action.SEND, context.pipes.send_pipes, context.pipes.reader_events) ] for action, pipes, events in cases: - path = mock_context.emit_pipe_creation_event(action, mkfifo) + path = context.emit_pipe_creation_event(action, mkfifo) assert events[-1] == path assert path in pipes.keys() remove(path) - mock_context.pipes.observer.on_any_event(InotifyFileDeletedEvent(path)) + context.pipes.observer.on_any_event(InotifyFileDeletedEvent(path)) assert path not in pipes.keys() -def test_handle_message(mock_context): - path = mock_context.emit_pipe_creation_event(Action.SEND, mkfifo) +def test_handle_message(context): + path = context.emit_pipe_creation_event(Action.SEND, mkfifo) payload = {'key': token_urlsafe(16)} - mock_context.pipes.send_pipes[path].handle_message(dumps(payload)) - assert mock_context.pipes.connector.messages[-1] == payload - mock_context.pipes.send_pipes[path].handle_message(token_urlsafe(32)) - assert len(mock_context.pipes.connector.messages) == 1 + context.pipes.send_pipes[path].handle_message(dumps(payload)) + assert context.pipes.connector.messages[-1] == payload + context.pipes.send_pipes[path].handle_message(token_urlsafe(32)) + assert len(context.pipes.connector.messages) == 1 -def test_broadcast(mock_context): +def test_broadcast(context): paths = [ - mock_context.emit_pipe_creation_event(Action.RECV, mkfifo) + context.emit_pipe_creation_event(Action.RECV, mkfifo) for _ in range(32) ] payload = {'key': token_urlsafe(16)} - mock_context.pipes.broadcast(payload) + context.pipes.broadcast(payload) for path in paths: - assert mock_context.pipes.recv_pipes[path].messages[-1] == payload + assert context.pipes.recv_pipes[path].messages[-1] == payload -def test_inode_types(mock_context): +def test_inode_types(context): touch = lambda path: Path(path).touch() cases = [ - (Action.RECV, mock_context.pipes.recv_pipes, mkdir), - (Action.SEND, mock_context.pipes.send_pipes, mkdir), - (Action.RECV, mock_context.pipes.recv_pipes, touch), - (Action.SEND, mock_context.pipes.send_pipes, touch) + (Action.RECV, context.pipes.recv_pipes, mkdir), + (Action.SEND, context.pipes.send_pipes, mkdir), + (Action.RECV, context.pipes.recv_pipes, touch), + (Action.SEND, context.pipes.send_pipes, touch) ] for action, pipes, creator in cases: - path = mock_context.emit_pipe_creation_event(action, creator) + path = context.emit_pipe_creation_event(action, creator) assert path not in pipes.keys() - - -def test_build_reader_implemented(context): - with pytest.raises(NotImplementedError): - context.emit_pipe_creation_event(Action.SEND, mkfifo) From aa1112af0eb28609d019c1780b1c88f45142fc0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 11 Sep 2019 15:44:42 -0400 Subject: [PATCH 171/174] Refactor connecting existing pipes to a method --- .../pipe_io/pipe_connector/pipe_connector.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tfw/components/pipe_io/pipe_connector/pipe_connector.py b/tfw/components/pipe_io/pipe_connector/pipe_connector.py index fa423da..06ba4ab 100644 --- a/tfw/components/pipe_io/pipe_connector/pipe_connector.py +++ b/tfw/components/pipe_io/pipe_connector/pipe_connector.py @@ -18,11 +18,7 @@ class PipeConnector: self.observer = self.build_observer(path) self.observer.on_any_event = self._on_any_event self.observer.start() - for node in listdir(path): - node_path = join(path, node) - if self._is_pipe(node_path): - self._create_pipe(node_path) - LOG.debug('Connected to existing pipe "%s"', node_path) + self._connect_existing_pipes(path) def build_observer(self, path): # pylint: disable=no-self-use if not exists(path): @@ -31,6 +27,13 @@ class PipeConnector: raise ValueError('Path does not exist or is not accessible.') return InotifyObserver(path, patterns=[f'*{self.reader_pattern}*', f'*{self.writer_pattern}*']) + def _connect_existing_pipes(self, path): + for node in listdir(path): + node_path = join(path, node) + if self._is_pipe(node_path): + self._create_pipe(node_path) + LOG.debug('Connected to existing pipe "%s"', node_path) + def _on_any_event(self, event): path = event.src_path if self._is_pipe(path) and isinstance(event, InotifyFileCreatedEvent): From 6c5699cff9c1f1892857601a47b3b0bcf53fdde9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 11 Sep 2019 20:28:18 -0400 Subject: [PATCH 172/174] Make unused but overridden public method protected --- tfw/components/pipe_io/pipe_connector/pipe_connector.py | 4 ++-- .../pipe_io/pipe_connector/test_proxy_pipe_connector.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tfw/components/pipe_io/pipe_connector/pipe_connector.py b/tfw/components/pipe_io/pipe_connector/pipe_connector.py index 06ba4ab..6f76e0e 100644 --- a/tfw/components/pipe_io/pipe_connector/pipe_connector.py +++ b/tfw/components/pipe_io/pipe_connector/pipe_connector.py @@ -15,12 +15,12 @@ class PipeConnector: def __init__(self, path): self.recv_pipes, self.send_pipes = {}, {} - self.observer = self.build_observer(path) + self.observer = self._build_observer(path) self.observer.on_any_event = self._on_any_event self.observer.start() self._connect_existing_pipes(path) - def build_observer(self, path): # pylint: disable=no-self-use + def _build_observer(self, path): # pylint: disable=no-self-use if not exists(path): mkdir(path) if not access(path, R_OK): diff --git a/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py b/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py index e49f7cc..989c5bf 100644 --- a/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py +++ b/tfw/components/pipe_io/pipe_connector/test_proxy_pipe_connector.py @@ -43,7 +43,7 @@ class MockPipeConnector(ProxyPipeConnector): self.reader_events, self.writer_events = [], [] super().__init__(path, connector) - def build_observer(self, path): + def _build_observer(self, path): return MockObserver() def build_reader(self, path): From 56620c4b5f76c0950a3c5c6b592ed7c24aeb5367 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 5 Sep 2019 18:16:53 +0200 Subject: [PATCH 173/174] Copy frontend as a multistage artifact --- Dockerfile | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index d14cc40..db441a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,14 @@ +FROM tfw-frontend:latest as frontend FROM avatao/debian:buster -RUN curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - &&\ - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - &&\ - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list &&\ - apt-get update &&\ +RUN apt-get update &&\ apt-get install -y --no-install-recommends \ - nodejs \ - yarn \ supervisor \ libzmq5 \ nginx \ jq \ - gettext-base &&\ - rm -rf /var/lib/apt/lists/* &&\ + gettext-base &&\ + rm -rf /var/lib/apt/lists/* &&\ ln -sf /bin/bash /bin/sh COPY requirements.txt /tmp @@ -59,11 +55,11 @@ COPY nginx/default.conf ${TFW_NGINX_DEFAULT} COPY nginx/components/ ${TFW_NGINX_COMPONENTS} COPY tfw ${TFW_LIB_DIR}/tfw COPY supervisor/tfw_server.py ${TFW_SERVER_DIR}/ +COPY --from=frontend /srv/dist ${TFW_FRONTEND_DIR} VOLUME ["${TFW_LOGS_DIR}", "${TFW_PIPES_DIR}"] ONBUILD ARG BUILD_CONTEXT="solvable" -ONBUILD ARG NOFRONTEND="" ONBUILD COPY ${BUILD_CONTEXT}/nginx/ ${TFW_NGINX_COMPONENTS} ONBUILD COPY ${BUILD_CONTEXT}/supervisor/ ${TFW_SUPERVISORD_COMPONENTS} @@ -73,10 +69,5 @@ ONBUILD RUN for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do done ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx", "${TFW_LIB_DIR}/tfw"] -ONBUILD COPY ${BUILD_CONTEXT}/frontend /data/ -ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn install --frozen-lockfile || : -ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn build --no-progress || : -ONBUILD RUN test -z "${NOFRONTEND}" && mv /data/dist ${TFW_FRONTEND_DIR} && rm -rf /data || : - ENTRYPOINT ["/bin/init", "--"] CMD exec supervisord --nodaemon --configuration ${TFW_SUPERVISORD_CONF} From 70cd4c51cbd3109301724c2aa39a6e6872dd9280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 12 Sep 2019 13:40:13 -0400 Subject: [PATCH 174/174] Use new frontend image tag from DockerHub --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index db441a7..a0f1cd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM tfw-frontend:latest as frontend +FROM avatao/frontend-tutorial-framework:chausie-20190912 as frontend FROM avatao/debian:buster RUN apt-get update &&\