From 52b2adb9c4c5b73a4bb22e9bd8ff03409fdc16e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 24 Jul 2018 11:34:51 +0200 Subject: [PATCH 01/48] Add missing legal boilerplate to YamlFSM --- lib/tfw/yaml_fsm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/tfw/yaml_fsm.py b/lib/tfw/yaml_fsm.py index 63070d2..f454c22 100644 --- a/lib/tfw/yaml_fsm.py +++ b/lib/tfw/yaml_fsm.py @@ -1,3 +1,6 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + from subprocess import Popen, run from functools import partial, singledispatch from contextlib import suppress From 8c6a14cef5b0c6175e490d2020a04cc4c5fb616d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 24 Jul 2018 11:40:33 +0200 Subject: [PATCH 02/48] Move fsm stuff to a separate directory in lib --- lib/tfw/__init__.py | 4 +--- lib/tfw/fsm/__init__.py | 6 ++++++ lib/tfw/{ => fsm}/fsm_base.py | 0 lib/tfw/{ => fsm}/linear_fsm.py | 0 lib/tfw/{ => fsm}/yaml_fsm.py | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 lib/tfw/fsm/__init__.py rename lib/tfw/{ => fsm}/fsm_base.py (100%) rename lib/tfw/{ => fsm}/linear_fsm.py (100%) rename lib/tfw/{ => fsm}/yaml_fsm.py (99%) diff --git a/lib/tfw/__init__.py b/lib/tfw/__init__.py index 04d934d..9c67327 100644 --- a/lib/tfw/__init__.py +++ b/lib/tfw/__init__.py @@ -2,6 +2,4 @@ # All Rights Reserved. See LICENSE file for details. from .event_handler_base import EventHandlerBase, FSMAwareEventHandler, BroadcastingEventHandler -from .fsm_base import FSMBase -from .linear_fsm import LinearFSM -from .yaml_fsm import YamlFSM +from .fsm import FSMBase, LinearFSM, YamlFSM diff --git a/lib/tfw/fsm/__init__.py b/lib/tfw/fsm/__init__.py new file mode 100644 index 0000000..11a7a58 --- /dev/null +++ b/lib/tfw/fsm/__init__.py @@ -0,0 +1,6 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from .fsm_base import FSMBase +from .linear_fsm import LinearFSM +from .yaml_fsm import YamlFSM diff --git a/lib/tfw/fsm_base.py b/lib/tfw/fsm/fsm_base.py similarity index 100% rename from lib/tfw/fsm_base.py rename to lib/tfw/fsm/fsm_base.py diff --git a/lib/tfw/linear_fsm.py b/lib/tfw/fsm/linear_fsm.py similarity index 100% rename from lib/tfw/linear_fsm.py rename to lib/tfw/fsm/linear_fsm.py diff --git a/lib/tfw/yaml_fsm.py b/lib/tfw/fsm/yaml_fsm.py similarity index 99% rename from lib/tfw/yaml_fsm.py rename to lib/tfw/fsm/yaml_fsm.py index f454c22..38f7a8e 100644 --- a/lib/tfw/yaml_fsm.py +++ b/lib/tfw/fsm/yaml_fsm.py @@ -9,7 +9,7 @@ import yaml import jinja2 from transitions import State -from tfw import FSMBase +from .fsm_base import FSMBase class YamlFSM(FSMBase): From d718b6425e2ec8b514fc4eebe2e220c09fbbab22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 24 Jul 2018 14:53:17 +0200 Subject: [PATCH 03/48] Refactor FSMAware part from FSMAwareEH to a separate class --- lib/tfw/event_handler_base.py | 39 +++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base.py index aaf408c..de170f0 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base.py @@ -106,25 +106,17 @@ class EventHandlerBase(ABC): pass -class FSMAwareEventHandler(EventHandlerBase, ABC): - # pylint: disable=abstract-method - """ - Abstract base class for EventHandlers which automatically - keep track of the state of the TFW FSM. - """ - def __init__(self, key): - super().__init__(key) - self.subscribe('fsm_update') +class FSMAware: + def __init__(self): self.fsm_state = None self.in_accepted_state = False self._auth_key = KeyManager().auth_key - def dispatch_handling(self, message): - if message['key'] == 'fsm_update': - if verify_message(self._auth_key, message): - self._handle_fsm_update(message) - return None - return super().dispatch_handling(message) + def update_fsm_data(self, message): + if message['key'] == 'fsm_update' and verify_message(self._auth_key, message): + self._handle_fsm_update(message) + return True + return False def _handle_fsm_update(self, message): try: @@ -144,6 +136,23 @@ class FSMAwareEventHandler(EventHandlerBase, ABC): pass +class FSMAwareEventHandler(EventHandlerBase, FSMAware, ABC): + # pylint: disable=abstract-method + """ + Abstract base class for EventHandlers which automatically + keep track of the state of the TFW FSM. + """ + def __init__(self, key): + EventHandlerBase.__init__(self, key) + FSMAware.__init__(self) + self.subscribe('fsm_update') + + def dispatch_handling(self, message): + if self.update_fsm_data(message): + return None + return super().dispatch_handling(message) + + class BroadcastingEventHandler(EventHandlerBase, ABC): # pylint: disable=abstract-method """ From a6b7fa04ab0dea2e2fe70d66b49006aeb795e502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 24 Jul 2018 17:16:57 +0200 Subject: [PATCH 04/48] Rework fsm_update API --- .../components/fsm_managing_event_handler.py | 21 ++++++++++--------- lib/tfw/event_handler_base.py | 11 +++++----- lib/tfw/fsm/fsm_base.py | 18 ++++++++++++++-- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/lib/tfw/components/fsm_managing_event_handler.py b/lib/tfw/components/fsm_managing_event_handler.py index 47761a0..39ec3ae 100644 --- a/lib/tfw/components/fsm_managing_event_handler.py +++ b/lib/tfw/components/fsm_managing_event_handler.py @@ -25,7 +25,7 @@ class FSMManagingEventHandler(EventHandlerBase): try: message = self.command_handlers[message['data']['command']](message) if message: - fsm_update_message = self._fsm_updater.generate_fsm_update() + fsm_update_message = self._fsm_updater.fsm_update sign_message(self.auth_key, message) sign_message(self.auth_key, fsm_update_message) self.server_connector.broadcast(fsm_update_message) @@ -52,23 +52,24 @@ class FSMUpdater: def __init__(self, fsm): self.fsm = fsm - def generate_fsm_update(self): + @property + def fsm_update(self): return { 'key': 'fsm_update', - 'data': self.get_fsm_state_and_transitions() + 'data': self.fsm_update_data } - def get_fsm_state_and_transitions(self): - state = self.fsm.state + @property + def fsm_update_data(self): valid_transitions = [ {'trigger': trigger} for trigger in self.fsm.get_triggers(self.fsm.state) ] - last_trigger = self.fsm.trigger_history[-1] if self.fsm.trigger_history else None - in_accepted_state = state in self.fsm.accepted_states + last_fsm_event = self.fsm.event_log[-1] + last_fsm_event['timestamp'] = last_fsm_event['timestamp'].isoformat() return { - 'current_state': state, + 'current_state': self.fsm.state, 'valid_transitions': valid_transitions, - 'last_trigger': last_trigger, - 'in_accepted_state': in_accepted_state + 'in_accepted_state': self.fsm.in_accepted_state, + 'last_event': last_fsm_event } diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base.py index de170f0..4c7ad06 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base.py @@ -109,7 +109,7 @@ class EventHandlerBase(ABC): class FSMAware: def __init__(self): self.fsm_state = None - self.in_accepted_state = False + self.fsm_in_accepted_state = False self._auth_key = KeyManager().auth_key def update_fsm_data(self, message): @@ -121,17 +121,18 @@ class FSMAware: def _handle_fsm_update(self, message): try: new_state = message['data']['current_state'] - trigger = message['data']['last_trigger'] if self.fsm_state != new_state: - self.handle_fsm_step(self.fsm_state, new_state, trigger) + self.handle_fsm_step(**(message['data'])) self.fsm_state = new_state - self.in_accepted_state = message['data']['in_accepted_state'] + self.fsm_in_accepted_state = message['data']['in_accepted_state'] except KeyError: LOG.error('Invalid fsm_update message received!') - def handle_fsm_step(self, from_state, to_state, trigger): + def handle_fsm_step(self, **kwargs): """ Called in case the TFW FSM has stepped. + + :param kwargs: fsm_update 'data' field """ pass diff --git a/lib/tfw/fsm/fsm_base.py b/lib/tfw/fsm/fsm_base.py index 7ae8157..e33ae33 100644 --- a/lib/tfw/fsm/fsm_base.py +++ b/lib/tfw/fsm/fsm_base.py @@ -2,6 +2,7 @@ # All Rights Reserved. See LICENSE file for details. from collections import defaultdict +from datetime import datetime from transitions import Machine, MachineError @@ -24,7 +25,7 @@ class FSMBase(Machine, CallbackMixin): def __init__(self, initial=None, accepted_states=None): self.accepted_states = accepted_states or [self.states[-1].name] self.trigger_predicates = defaultdict(list) - self.trigger_history = [] + self.event_log = [] Machine.__init__( self, @@ -60,9 +61,22 @@ class FSMBase(Machine, CallbackMixin): if all(predicate_results): try: + from_state = self.state self.trigger(trigger) - self.trigger_history.append(trigger) + self.update_event_log(from_state, trigger) return True except (AttributeError, MachineError): LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger) return False + + def update_event_log(self, from_state, trigger): + self.event_log.append({ + 'from_state': from_state, + 'to_state': self.state, + 'trigger': trigger, + 'timestamp': datetime.utcnow() + }) + + @property + def in_accepted_state(self): + return self.state in self.accepted_states From fe79b598a7da456bc608ea116ff121da618ba371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 24 Jul 2018 17:18:04 +0200 Subject: [PATCH 05/48] Update major version due to API breakage --- .drone.yml | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 9dbc1f4..3a09a10 100644 --- a/.drone.yml +++ b/.drone.yml @@ -10,4 +10,4 @@ pipeline: - docker push eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG} when: event: 'tag' - branch: refs/tags/mainecoon-20* + branch: refs/tags/ocicat-20* diff --git a/VERSION b/VERSION index c1385d5..4ed6a03 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -mainecoon +ocicat From 5715c57ebcebd7b998cd9ccfc723d8c164035c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 25 Jul 2018 15:31:03 +0200 Subject: [PATCH 06/48] Move FSMAware to a separate file in networking --- lib/tfw/event_handler_base.py | 34 ++----------------------- lib/tfw/networking/__init__.py | 1 + lib/tfw/networking/fsm_aware.py | 45 +++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 32 deletions(-) create mode 100644 lib/tfw/networking/fsm_aware.py diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base.py index 4c7ad06..15d46bc 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base.py @@ -3,8 +3,9 @@ from abc import ABC, abstractmethod +from tfw.networking import FSMAware from tfw.networking.event_handlers import ServerConnector -from tfw.crypto import message_checksum, KeyManager, verify_message +from tfw.crypto import message_checksum from tfw.config.logs import logging LOG = logging.getLogger(__name__) @@ -106,37 +107,6 @@ class EventHandlerBase(ABC): pass -class FSMAware: - def __init__(self): - self.fsm_state = None - self.fsm_in_accepted_state = False - self._auth_key = KeyManager().auth_key - - def update_fsm_data(self, message): - if message['key'] == 'fsm_update' and verify_message(self._auth_key, message): - self._handle_fsm_update(message) - return True - return False - - def _handle_fsm_update(self, message): - try: - new_state = message['data']['current_state'] - if self.fsm_state != new_state: - self.handle_fsm_step(**(message['data'])) - self.fsm_state = new_state - self.fsm_in_accepted_state = message['data']['in_accepted_state'] - except KeyError: - LOG.error('Invalid fsm_update message received!') - - def handle_fsm_step(self, **kwargs): - """ - Called in case the TFW FSM has stepped. - - :param kwargs: fsm_update 'data' field - """ - pass - - class FSMAwareEventHandler(EventHandlerBase, FSMAware, ABC): # pylint: disable=abstract-method """ diff --git a/lib/tfw/networking/__init__.py b/lib/tfw/networking/__init__.py index c1f3eb4..0f038b0 100644 --- a/lib/tfw/networking/__init__.py +++ b/lib/tfw/networking/__init__.py @@ -7,3 +7,4 @@ from .zmq_connector_base import ZMQConnectorBase from .message_sender import MessageSender from .event_handlers.server_connector import ServerUplinkConnector as TFWServerConnector from .server.tfw_server import TFWServer +from .fsm_aware import FSMAware diff --git a/lib/tfw/networking/fsm_aware.py b/lib/tfw/networking/fsm_aware.py new file mode 100644 index 0000000..e666d8c --- /dev/null +++ b/lib/tfw/networking/fsm_aware.py @@ -0,0 +1,45 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from tfw.crypto import KeyManager, verify_message + +from tfw.config.logs import logging + +LOG = logging.getLogger(__name__) + + +class FSMAware: + """ + Base class for stuff that has to be aware of the framework FSM. + This is done by processing 'fsm_update' messages. + """ + def __init__(self): + self.fsm_state = None + self.fsm_in_accepted_state = False + self.fsm_last_update = None + self._auth_key = KeyManager().auth_key + + def update_fsm_data(self, message): + if message['key'] == 'fsm_update' and verify_message(self._auth_key, message): + self._handle_fsm_update(message) + return True + return False + + def _handle_fsm_update(self, message): + try: + new_state = message['data']['current_state'] + if self.fsm_state != new_state: + self.handle_fsm_step(**(message['data'])) + self.fsm_state = new_state + self.fsm_in_accepted_state = message['data']['in_accepted_state'] + self.fsm_last_update = message['data'] + except KeyError: + LOG.error('Invalid fsm_update message received!') + + def handle_fsm_step(self, **kwargs): + """ + Called in case the TFW FSM has stepped. + + :param kwargs: fsm_update 'data' field + """ + pass From ca09e868f16eb48d4070a283cb4cbcd515e3dc5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 25 Jul 2018 15:45:54 +0200 Subject: [PATCH 07/48] Fix ZMQWSProxy not executing filters/callbacks on nested messages --- .../networking/server/zmq_websocket_proxy.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/tfw/networking/server/zmq_websocket_proxy.py b/lib/tfw/networking/server/zmq_websocket_proxy.py index f456d94..cb43a30 100644 --- a/lib/tfw/networking/server/zmq_websocket_proxy.py +++ b/lib/tfw/networking/server/zmq_websocket_proxy.py @@ -105,14 +105,9 @@ class TFWProxy: raise ValueError('Invalid TFW message format!') def __call__(self, message): - try: - self.proxy_filters._execute_callbacks(message) - except ValueError: - LOG.exception('Invalid TFW message received!') + if not self.filter_and_execute_callbacks(message): return - self.proxy_callbacks._execute_callbacks(message) - if message['key'] not in self.keyhandlers: self.to_destination(message) else: @@ -122,13 +117,26 @@ class TFWProxy: except KeyError: LOG.error('Invalid "%s" message format! Ignoring.', handler.__name__) + def filter_and_execute_callbacks(self, message): + try: + self.proxy_filters._execute_callbacks(message) + self.proxy_callbacks._execute_callbacks(message) + return True + except ValueError: + LOG.exception('Invalid TFW message received!') + return False + def mirror(self, message): message = message['data'] + if not self.filter_and_execute_callbacks(message): + return LOG.debug('Mirroring message: %s', message) self.to_source(message) def broadcast(self, message): message = message['data'] + if not self.filter_and_execute_callbacks(message): + return LOG.debug('Broadcasting message: %s', message) self.to_source(message) self.to_destination(message) From 7fb5a3783185119a24a4f982c90dbb762247b643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 25 Jul 2018 15:46:39 +0200 Subject: [PATCH 08/48] Make TFWServer FSMAware --- lib/tfw/networking/server/tfw_server.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/tfw/networking/server/tfw_server.py b/lib/tfw/networking/server/tfw_server.py index c0d98f5..f4a83a6 100644 --- a/lib/tfw/networking/server/tfw_server.py +++ b/lib/tfw/networking/server/tfw_server.py @@ -9,20 +9,23 @@ from tornado.web import Application from tfw.networking.event_handlers import ServerUplinkConnector from tfw.networking.server import EventHandlerConnector from tfw.networking import MessageSender +from tfw.networking.fsm_aware import FSMAware from tfw.crypto import KeyManager, verify_message, sign_message from tfw.config.logs import logging + from .zmq_websocket_proxy import ZMQWebSocketProxy LOG = logging.getLogger(__name__) -class TFWServer: +class TFWServer(FSMAware): """ This class handles the proxying of messages between the frontend and event handers. It proxies messages from the "/ws" route to all event handlers subscribed to a ZMQ SUB socket. """ def __init__(self): + super().__init__() self._event_handler_connector = EventHandlerConnector() self._uplink_connector = ServerUplinkConnector() self._auth_key = KeyManager().auth_key @@ -30,7 +33,11 @@ class TFWServer: self.application = Application([( r'/ws', ZMQWebSocketProxy, { 'event_handler_connector': self._event_handler_connector, - 'message_handlers': [self.handle_trigger, self.handle_recover], + 'message_handlers': [ + self.handle_trigger, + self.handle_recover, + self.handle_fsm_update + ], 'frontend_message_handlers': [self.save_frontend_messages] })]) @@ -55,6 +62,9 @@ class TFWServer: self._frontend_messages.replay_messages(self._uplink_connector) self._frontend_messages.clear() + def handle_fsm_update(self, message): + self.update_fsm_data(message) + def save_frontend_messages(self, message): self._frontend_messages.save_message(message) From 7a670f37f2011a87c9a7faf092e0b47ceb3b31d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 26 Jul 2018 13:59:06 +0200 Subject: [PATCH 09/48] Resolve Python circular import hell - hopefully forever --- lib/envvars/__init__.py | 2 +- lib/tfw/__init__.py | 3 -- lib/tfw/components/directory_monitor.py | 6 +-- .../directory_monitoring_event_handler.py | 6 +-- .../components/fsm_managing_event_handler.py | 2 +- lib/tfw/components/history_monitor.py | 5 ++- lib/tfw/components/ide_event_handler.py | 6 +-- lib/tfw/components/log_monitor.py | 7 +-- .../log_monitoring_event_handler.py | 6 +-- .../process_managing_event_handler.py | 6 +-- lib/tfw/components/terminal_event_handler.py | 4 +- lib/tfw/crypto.py | 4 +- lib/tfw/decorators/__init__.py | 3 -- lib/tfw/event_handler_base/__init__.py | 6 +++ .../boradcasting_event_handler.py | 30 +++++++++++++ .../event_handler_base.py | 44 +------------------ .../fsm_aware_event_handler.py | 24 ++++++++++ lib/tfw/fsm/fsm_base.py | 2 +- lib/tfw/fsm/linear_fsm.py | 2 +- lib/tfw/fsm/yaml_fsm.py | 2 +- lib/tfw/mixins/__init__.py | 5 --- lib/tfw/mixins/callback_mixin.py | 2 +- lib/tfw/mixins/observer_mixin.py | 2 +- lib/tfw/mixins/supervisor_mixin.py | 2 +- lib/tfw/networking/__init__.py | 4 -- lib/tfw/networking/event_handlers/__init__.py | 2 - .../event_handlers/server_connector.py | 4 +- lib/tfw/networking/message_sender.py | 2 +- lib/tfw/networking/server/__init__.py | 3 -- .../server/event_handler_connector.py | 3 +- lib/tfw/networking/server/tfw_server.py | 9 ++-- .../networking/server/zmq_websocket_proxy.py | 2 +- 32 files changed, 105 insertions(+), 105 deletions(-) create mode 100644 lib/tfw/event_handler_base/__init__.py create mode 100644 lib/tfw/event_handler_base/boradcasting_event_handler.py rename lib/tfw/{ => event_handler_base}/event_handler_base.py (68%) create mode 100644 lib/tfw/event_handler_base/fsm_aware_event_handler.py diff --git a/lib/envvars/__init__.py b/lib/envvars/__init__.py index d1ed0d4..46774d4 100644 --- a/lib/envvars/__init__.py +++ b/lib/envvars/__init__.py @@ -4,7 +4,7 @@ from collections import namedtuple from os import environ -from tfw.decorators import lazy_property +from tfw.decorators.lazy_property import lazy_property class LazyEnvironment: diff --git a/lib/tfw/__init__.py b/lib/tfw/__init__.py index 9c67327..db64b25 100644 --- a/lib/tfw/__init__.py +++ b/lib/tfw/__init__.py @@ -1,5 +1,2 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. - -from .event_handler_base import EventHandlerBase, FSMAwareEventHandler, BroadcastingEventHandler -from .fsm import FSMBase, LinearFSM, YamlFSM diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index c72a13e..f65c8ff 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -5,9 +5,9 @@ from functools import wraps from watchdog.events import FileSystemEventHandler as FileSystemWatchdogEventHandler -from tfw.networking.event_handlers import ServerUplinkConnector -from tfw.decorators import RateLimiter -from tfw.mixins import ObserverMixin +from tfw.networking.event_handlers.server_connector import ServerUplinkConnector +from tfw.decorators.rate_limiter import RateLimiter +from tfw.mixins.observer_mixin import ObserverMixin from tfw.config.logs import logging diff --git a/lib/tfw/components/directory_monitoring_event_handler.py b/lib/tfw/components/directory_monitoring_event_handler.py index 6f89e91..8d97022 100644 --- a/lib/tfw/components/directory_monitoring_event_handler.py +++ b/lib/tfw/components/directory_monitoring_event_handler.py @@ -3,10 +3,10 @@ from os.path import isdir, exists -from tfw import EventHandlerBase +from tfw.event_handler_base import EventHandlerBase +from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin +from tfw.components.directory_monitor import DirectoryMonitor from tfw.config.logs import logging -from tfw.mixins import MonitorManagerMixin -from .directory_monitor import DirectoryMonitor LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/fsm_managing_event_handler.py b/lib/tfw/components/fsm_managing_event_handler.py index 39ec3ae..376ce50 100644 --- a/lib/tfw/components/fsm_managing_event_handler.py +++ b/lib/tfw/components/fsm_managing_event_handler.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw import EventHandlerBase +from tfw.event_handler_base import EventHandlerBase from tfw.crypto import KeyManager, sign_message, verify_message from tfw.config.logs import logging diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index 9d259ae..410558a 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -8,8 +8,9 @@ from abc import ABC, abstractmethod from watchdog.events import PatternMatchingEventHandler -from tfw.mixins import CallbackMixin, ObserverMixin -from tfw.decorators import RateLimiter +from tfw.mixins.callback_mixin import CallbackMixin +from tfw.mixins.observer_mixin import ObserverMixin +from tfw.decorators.rate_limiter import RateLimiter class CallbackEventHandler(PatternMatchingEventHandler, ABC): diff --git a/lib/tfw/components/ide_event_handler.py b/lib/tfw/components/ide_event_handler.py index 2c698ad..05905be 100644 --- a/lib/tfw/components/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler.py @@ -6,10 +6,10 @@ from glob import glob from fnmatch import fnmatchcase from typing import Iterable -from tfw import EventHandlerBase -from tfw.mixins import MonitorManagerMixin +from tfw.event_handler_base import EventHandlerBase +from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin +from tfw.components.directory_monitor import DirectoryMonitor from tfw.config.logs import logging -from .directory_monitor import DirectoryMonitor LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/log_monitor.py b/lib/tfw/components/log_monitor.py index 2c6ade8..cf06a0a 100644 --- a/lib/tfw/components/log_monitor.py +++ b/lib/tfw/components/log_monitor.py @@ -6,9 +6,10 @@ from os.path import dirname from watchdog.events import PatternMatchingEventHandler as PatternMatchingWatchdogEventHandler -from tfw.networking.event_handlers import ServerUplinkConnector -from tfw.decorators import RateLimiter -from tfw.mixins import ObserverMixin, SupervisorLogMixin +from tfw.networking.event_handlers.server_connector import ServerUplinkConnector +from tfw.decorators.rate_limiter import RateLimiter +from tfw.mixins.observer_mixin import ObserverMixin +from tfw.mixins.supervisor_mixin import SupervisorLogMixin class LogMonitor(ObserverMixin): diff --git a/lib/tfw/components/log_monitoring_event_handler.py b/lib/tfw/components/log_monitoring_event_handler.py index c5dd84c..0bc7ab2 100644 --- a/lib/tfw/components/log_monitoring_event_handler.py +++ b/lib/tfw/components/log_monitoring_event_handler.py @@ -1,10 +1,10 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw import EventHandlerBase -from tfw.mixins import MonitorManagerMixin +from tfw.event_handler_base import EventHandlerBase +from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin +from tfw.components.log_monitor import LogMonitor from tfw.config.logs import logging -from .log_monitor import LogMonitor LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/process_managing_event_handler.py b/lib/tfw/components/process_managing_event_handler.py index 040995e..61b6c62 100644 --- a/lib/tfw/components/process_managing_event_handler.py +++ b/lib/tfw/components/process_managing_event_handler.py @@ -3,10 +3,10 @@ from xmlrpc.client import Fault as SupervisorFault -from tfw import EventHandlerBase -from tfw.mixins import SupervisorMixin, SupervisorLogMixin +from tfw.event_handler_base import EventHandlerBase +from tfw.mixins.supervisor_mixin import SupervisorMixin, SupervisorLogMixin +from tfw.components.directory_monitor import with_monitor_paused from tfw.config.logs import logging -from .directory_monitor import with_monitor_paused LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/components/terminal_event_handler.py index 79e96f7..27bbce7 100644 --- a/lib/tfw/components/terminal_event_handler.py +++ b/lib/tfw/components/terminal_event_handler.py @@ -1,11 +1,11 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw import EventHandlerBase +from tfw.event_handler_base import EventHandlerBase +from tfw.components.terminado_mini_server import TerminadoMiniServer from tfw.config import TFWENV from tfw.config.logs import logging from tao.config import TAOENV -from .terminado_mini_server import TerminadoMiniServer LOG = logging.getLogger(__name__) diff --git a/lib/tfw/crypto.py b/lib/tfw/crypto.py index af632d8..0b37893 100644 --- a/lib/tfw/crypto.py +++ b/lib/tfw/crypto.py @@ -14,8 +14,8 @@ from cryptography.hazmat.primitives.hashes import SHA256 from cryptography.hazmat.primitives.hmac import HMAC as _HMAC from cryptography.exceptions import InvalidSignature -from tfw.networking import message_bytes -from tfw.decorators import lazy_property +from tfw.networking.serialization import message_bytes +from tfw.decorators.lazy_property import lazy_property from tfw.config import TFWENV diff --git a/lib/tfw/decorators/__init__.py b/lib/tfw/decorators/__init__.py index ed79d7f..db64b25 100644 --- a/lib/tfw/decorators/__init__.py +++ b/lib/tfw/decorators/__init__.py @@ -1,5 +1,2 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. - -from .rate_limiter import RateLimiter -from .lazy_property import lazy_property diff --git a/lib/tfw/event_handler_base/__init__.py b/lib/tfw/event_handler_base/__init__.py new file mode 100644 index 0000000..fd50525 --- /dev/null +++ b/lib/tfw/event_handler_base/__init__.py @@ -0,0 +1,6 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from .event_handler_base import EventHandlerBase +from .boradcasting_event_handler import BroadcastingEventHandler +from .fsm_aware_event_handler import FSMAwareEventHandler diff --git a/lib/tfw/event_handler_base/boradcasting_event_handler.py b/lib/tfw/event_handler_base/boradcasting_event_handler.py new file mode 100644 index 0000000..59ee493 --- /dev/null +++ b/lib/tfw/event_handler_base/boradcasting_event_handler.py @@ -0,0 +1,30 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from abc import ABC + +from tfw.event_handler_base.event_handler_base import EventHandlerBase +from tfw.crypto import message_checksum + + +class BroadcastingEventHandler(EventHandlerBase, ABC): + # pylint: disable=abstract-method + """ + Abstract base class for EventHandlers which broadcast responses + and intelligently ignore their own broadcasted messages they receive. + """ + def __init__(self, key): + super().__init__(key) + self.own_message_hashes = [] + + def event_handler_callback(self, message): + message_hash = message_checksum(message) + + if message_hash in self.own_message_hashes: + self.own_message_hashes.remove(message_hash) + return + + response = self.dispatch_handling(message) + if response: + self.own_message_hashes.append(message_checksum(response)) + self.server_connector.broadcast(response) diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base/event_handler_base.py similarity index 68% rename from lib/tfw/event_handler_base.py rename to lib/tfw/event_handler_base/event_handler_base.py index 15d46bc..f676bac 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base/event_handler_base.py @@ -3,9 +3,7 @@ from abc import ABC, abstractmethod -from tfw.networking import FSMAware -from tfw.networking.event_handlers import ServerConnector -from tfw.crypto import message_checksum +from tfw.networking.event_handlers.server_connector import ServerConnector from tfw.config.logs import logging LOG = logging.getLogger(__name__) @@ -105,43 +103,3 @@ class EventHandlerBase(ABC): connections and stuff like that. """ pass - - -class FSMAwareEventHandler(EventHandlerBase, FSMAware, ABC): - # pylint: disable=abstract-method - """ - Abstract base class for EventHandlers which automatically - keep track of the state of the TFW FSM. - """ - def __init__(self, key): - EventHandlerBase.__init__(self, key) - FSMAware.__init__(self) - self.subscribe('fsm_update') - - def dispatch_handling(self, message): - if self.update_fsm_data(message): - return None - return super().dispatch_handling(message) - - -class BroadcastingEventHandler(EventHandlerBase, ABC): - # pylint: disable=abstract-method - """ - Abstract base class for EventHandlers which broadcast responses - and intelligently ignore their own broadcasted messages they receive. - """ - def __init__(self, key): - super().__init__(key) - self.own_message_hashes = [] - - def event_handler_callback(self, message): - message_hash = message_checksum(message) - - if message_hash in self.own_message_hashes: - self.own_message_hashes.remove(message_hash) - return - - response = self.dispatch_handling(message) - if response: - self.own_message_hashes.append(message_checksum(response)) - self.server_connector.broadcast(response) diff --git a/lib/tfw/event_handler_base/fsm_aware_event_handler.py b/lib/tfw/event_handler_base/fsm_aware_event_handler.py new file mode 100644 index 0000000..01d6360 --- /dev/null +++ b/lib/tfw/event_handler_base/fsm_aware_event_handler.py @@ -0,0 +1,24 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from abc import ABC + +from tfw.event_handler_base.event_handler_base import EventHandlerBase +from tfw.networking.fsm_aware import FSMAware + + +class FSMAwareEventHandler(EventHandlerBase, FSMAware, ABC): + # pylint: disable=abstract-method + """ + Abstract base class for EventHandlers which automatically + keep track of the state of the TFW FSM. + """ + def __init__(self, key): + EventHandlerBase.__init__(self, key) + FSMAware.__init__(self) + self.subscribe('fsm_update') + + def dispatch_handling(self, message): + if self.update_fsm_data(message): + return None + return super().dispatch_handling(message) diff --git a/lib/tfw/fsm/fsm_base.py b/lib/tfw/fsm/fsm_base.py index e33ae33..5b58ce3 100644 --- a/lib/tfw/fsm/fsm_base.py +++ b/lib/tfw/fsm/fsm_base.py @@ -6,7 +6,7 @@ from datetime import datetime from transitions import Machine, MachineError -from tfw.mixins import CallbackMixin +from tfw.mixins.callback_mixin import CallbackMixin from tfw.config.logs import logging LOG = logging.getLogger(__name__) diff --git a/lib/tfw/fsm/linear_fsm.py b/lib/tfw/fsm/linear_fsm.py index a053c30..cc5a0d1 100644 --- a/lib/tfw/fsm/linear_fsm.py +++ b/lib/tfw/fsm/linear_fsm.py @@ -3,7 +3,7 @@ from transitions import State -from .fsm_base import FSMBase +from tfw.fsm.fsm_base import FSMBase class LinearFSM(FSMBase): diff --git a/lib/tfw/fsm/yaml_fsm.py b/lib/tfw/fsm/yaml_fsm.py index 38f7a8e..af0b682 100644 --- a/lib/tfw/fsm/yaml_fsm.py +++ b/lib/tfw/fsm/yaml_fsm.py @@ -9,7 +9,7 @@ import yaml import jinja2 from transitions import State -from .fsm_base import FSMBase +from tfw.fsm.fsm_base import FSMBase class YamlFSM(FSMBase): diff --git a/lib/tfw/mixins/__init__.py b/lib/tfw/mixins/__init__.py index 58915ca..db64b25 100644 --- a/lib/tfw/mixins/__init__.py +++ b/lib/tfw/mixins/__init__.py @@ -1,7 +1,2 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. - -from .supervisor_mixin import SupervisorMixin, SupervisorLogMixin -from .callback_mixin import CallbackMixin -from .observer_mixin import ObserverMixin -from .monitor_manager_mixin import MonitorManagerMixin diff --git a/lib/tfw/mixins/callback_mixin.py b/lib/tfw/mixins/callback_mixin.py index 33ddb6d..8075f47 100644 --- a/lib/tfw/mixins/callback_mixin.py +++ b/lib/tfw/mixins/callback_mixin.py @@ -3,7 +3,7 @@ from functools import partial -from tfw.decorators import lazy_property +from tfw.decorators.lazy_property import lazy_property class CallbackMixin: diff --git a/lib/tfw/mixins/observer_mixin.py b/lib/tfw/mixins/observer_mixin.py index 9d8b5e2..d5415e5 100644 --- a/lib/tfw/mixins/observer_mixin.py +++ b/lib/tfw/mixins/observer_mixin.py @@ -3,7 +3,7 @@ from watchdog.observers import Observer -from tfw.decorators import lazy_property +from tfw.decorators.lazy_property import lazy_property class ObserverMixin: diff --git a/lib/tfw/mixins/supervisor_mixin.py b/lib/tfw/mixins/supervisor_mixin.py index 2238985..189d7cc 100644 --- a/lib/tfw/mixins/supervisor_mixin.py +++ b/lib/tfw/mixins/supervisor_mixin.py @@ -6,7 +6,7 @@ from xmlrpc.client import Fault as SupervisorFault from contextlib import suppress from os import remove -from tfw.decorators import lazy_property +from tfw.decorators.lazy_property import lazy_property from tfw.config import TFWENV diff --git a/lib/tfw/networking/__init__.py b/lib/tfw/networking/__init__.py index 0f038b0..500dd7a 100644 --- a/lib/tfw/networking/__init__.py +++ b/lib/tfw/networking/__init__.py @@ -1,10 +1,6 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from .serialization import serialize_tfw_msg, deserialize_tfw_msg -from .serialization import with_deserialize_tfw_msg, message_bytes -from .zmq_connector_base import ZMQConnectorBase from .message_sender import MessageSender from .event_handlers.server_connector import ServerUplinkConnector as TFWServerConnector from .server.tfw_server import TFWServer -from .fsm_aware import FSMAware diff --git a/lib/tfw/networking/event_handlers/__init__.py b/lib/tfw/networking/event_handlers/__init__.py index b3ad530..db64b25 100644 --- a/lib/tfw/networking/event_handlers/__init__.py +++ b/lib/tfw/networking/event_handlers/__init__.py @@ -1,4 +1,2 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. - -from .server_connector import ServerConnector, ServerUplinkConnector, ServerDownlinkConnector diff --git a/lib/tfw/networking/event_handlers/server_connector.py b/lib/tfw/networking/event_handlers/server_connector.py index 612bb8d..7d24803 100644 --- a/lib/tfw/networking/event_handlers/server_connector.py +++ b/lib/tfw/networking/event_handlers/server_connector.py @@ -6,8 +6,8 @@ from functools import partial import zmq from zmq.eventloop.zmqstream import ZMQStream -from tfw.networking import serialize_tfw_msg, with_deserialize_tfw_msg -from tfw.networking import ZMQConnectorBase +from tfw.networking.zmq_connector_base import ZMQConnectorBase +from tfw.networking.serialization import serialize_tfw_msg, with_deserialize_tfw_msg from tfw.config import TFWENV from tfw.config.logs import logging diff --git a/lib/tfw/networking/message_sender.py b/lib/tfw/networking/message_sender.py index 378ad91..58eb1c9 100644 --- a/lib/tfw/networking/message_sender.py +++ b/lib/tfw/networking/message_sender.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw.networking.event_handlers import ServerUplinkConnector +from tfw.networking.event_handlers.server_connector import ServerUplinkConnector class MessageSender: diff --git a/lib/tfw/networking/server/__init__.py b/lib/tfw/networking/server/__init__.py index eb9adde..db64b25 100644 --- a/lib/tfw/networking/server/__init__.py +++ b/lib/tfw/networking/server/__init__.py @@ -1,5 +1,2 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. - -from .event_handler_connector import EventHandlerConnector, EventHandlerUplinkConnector, EventHandlerDownlinkConnector -from .tfw_server import TFWServer diff --git a/lib/tfw/networking/server/event_handler_connector.py b/lib/tfw/networking/server/event_handler_connector.py index c4c6338..5075c56 100644 --- a/lib/tfw/networking/server/event_handler_connector.py +++ b/lib/tfw/networking/server/event_handler_connector.py @@ -4,7 +4,8 @@ import zmq from zmq.eventloop.zmqstream import ZMQStream -from tfw.networking import ZMQConnectorBase, serialize_tfw_msg, with_deserialize_tfw_msg +from tfw.networking.zmq_connector_base import ZMQConnectorBase +from tfw.networking.serialization import serialize_tfw_msg, with_deserialize_tfw_msg from tfw.config import TFWENV from tfw.config.logs import logging diff --git a/lib/tfw/networking/server/tfw_server.py b/lib/tfw/networking/server/tfw_server.py index f4a83a6..fcea107 100644 --- a/lib/tfw/networking/server/tfw_server.py +++ b/lib/tfw/networking/server/tfw_server.py @@ -6,15 +6,14 @@ from contextlib import suppress from tornado.web import Application -from tfw.networking.event_handlers import ServerUplinkConnector -from tfw.networking.server import EventHandlerConnector -from tfw.networking import MessageSender +from tfw.networking.server.zmq_websocket_proxy import ZMQWebSocketProxy +from tfw.networking.event_handlers.server_connector import ServerUplinkConnector +from tfw.networking.server.event_handler_connector import EventHandlerConnector +from tfw.networking.message_sender import MessageSender from tfw.networking.fsm_aware import FSMAware from tfw.crypto import KeyManager, verify_message, sign_message from tfw.config.logs import logging -from .zmq_websocket_proxy import ZMQWebSocketProxy - LOG = logging.getLogger(__name__) diff --git a/lib/tfw/networking/server/zmq_websocket_proxy.py b/lib/tfw/networking/server/zmq_websocket_proxy.py index cb43a30..6df7c9a 100644 --- a/lib/tfw/networking/server/zmq_websocket_proxy.py +++ b/lib/tfw/networking/server/zmq_websocket_proxy.py @@ -5,7 +5,7 @@ import json from tornado.websocket import WebSocketHandler -from tfw.mixins import CallbackMixin +from tfw.mixins.callback_mixin import CallbackMixin from tfw.config.logs import logging LOG = logging.getLogger(__name__) From 3fad818b2b61b214eab27aad3fd2907561cb9118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 26 Jul 2018 16:29:20 +0200 Subject: [PATCH 10/48] Add pylint pre-push hooks --- .git-hooks/apply_hooks.sh | 22 ++++++++++++++++++++++ .git-hooks/pre-push.sh | 18 ++++++++++++++++++ .pylintrc | 6 ++++++ 3 files changed, 46 insertions(+) create mode 100755 .git-hooks/apply_hooks.sh create mode 100755 .git-hooks/pre-push.sh diff --git a/.git-hooks/apply_hooks.sh b/.git-hooks/apply_hooks.sh new file mode 100755 index 0000000..f95f8f4 --- /dev/null +++ b/.git-hooks/apply_hooks.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -eu +set -o pipefail +set -o errtrace +shopt -s expand_aliases + +[ "$(uname)" == "Darwin" ] && alias readlink="greadlink" || : + +GREEN='\033[0;32m' +NC='\033[0m' + + +here="$(dirname "$(readlink -f "$0")")" +cd "${here}/../.git/hooks" + +rm -f pre-push pre-commit || : +prepush_script="../../.git-hooks/pre-push.sh" +precommit_script="../../.git-hooks/pre-commit.sh" +[ -f "${prepush_script}" ] && ln -s "${prepush_script}" pre-push +[ -f "${precommit_script}" ] && ln -s "${precommit_script}" pre-commit + +echo -e "\n${GREEN}Done! Hooks applied, you can start committing and pushing!${NC}\n" diff --git a/.git-hooks/pre-push.sh b/.git-hooks/pre-push.sh new file mode 100755 index 0000000..e2025ac --- /dev/null +++ b/.git-hooks/pre-push.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -eu +set -o pipefail +set -o errtrace +shopt -s expand_aliases + +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' + + +echo -e "Running pylint...\n" +if pylint lib; then + echo -e "\n${GREEN}Pylint found no errors!${NC}\n" +else + echo -e "\n${RED}Pylint failed with errors${NC}\n" + exit 1 +fi diff --git a/.pylintrc b/.pylintrc index f36e237..fd195bc 100644 --- a/.pylintrc +++ b/.pylintrc @@ -3,3 +3,9 @@ ignored-modules = zmq max-line-length = 120 disable = missing-docstring, too-few-public-methods, invalid-name + +[SIMILARITIES] + +ignore-comments=yes +ignore-docstrings=yes +ignore-imports=yes From 3d3328f835788d609fbd804a0e7b4f1ea089f0df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 27 Jul 2018 13:31:54 +0200 Subject: [PATCH 11/48] Fix bad indentation --- lib/tfw/fsm/yaml_fsm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/fsm/yaml_fsm.py b/lib/tfw/fsm/yaml_fsm.py index af0b682..d6c266d 100644 --- a/lib/tfw/fsm/yaml_fsm.py +++ b/lib/tfw/fsm/yaml_fsm.py @@ -48,7 +48,7 @@ class YamlFSM(FSMBase): partial( command_statuscode_is_zero, predicate - ) + ) ) with suppress(KeyError): From 8df196f258b057d0424e4daa9d5000a3d313baf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 27 Jul 2018 14:00:07 +0200 Subject: [PATCH 12/48] Add note on step_next trigger to LinearFSM --- lib/tfw/fsm/linear_fsm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/tfw/fsm/linear_fsm.py b/lib/tfw/fsm/linear_fsm.py index cc5a0d1..e515d8f 100644 --- a/lib/tfw/fsm/linear_fsm.py +++ b/lib/tfw/fsm/linear_fsm.py @@ -12,7 +12,8 @@ class LinearFSM(FSMBase): This is a state machine for challenges with linear progression, consisting of a number of steps specified in the constructor. It automatically sets up 2 actions (triggers) between states as such: - (0) -- step_1 --> (1) -- step_2 --> (2) -- step_3 --> (3) ... and so on + (0) -- step_1 --> (1) -- step_2 --> (2) -- step_3 --> (3) ... + (0) -- step_next --> (1) -- step_next --> (2) -- step_next --> (3) ... """ def __init__(self, number_of_steps): self.states = [State(name=str(index)) for index in range(number_of_steps)] From 732b896d17a7c69e997f49fa88c466402aa239ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 27 Jul 2018 14:00:45 +0200 Subject: [PATCH 13/48] Add docstrings to YamlFSM --- lib/tfw/fsm/yaml_fsm.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/tfw/fsm/yaml_fsm.py b/lib/tfw/fsm/yaml_fsm.py index d6c266d..fbfc542 100644 --- a/lib/tfw/fsm/yaml_fsm.py +++ b/lib/tfw/fsm/yaml_fsm.py @@ -13,7 +13,17 @@ from tfw.fsm.fsm_base import FSMBase class YamlFSM(FSMBase): + """ + This is a state machine capable of building itself from a YAML config file. + """ def __init__(self, config_file, jinja2_variables=None): + """ + :param config_file: path of the YAML file + :param jinja2_variables: dict containing jinja2 variables + or str with filename of YAML file to + parse and use as dict. + jinja2 support is disabled if this is None + """ self.config = ConfigParser(config_file, jinja2_variables).config self.setup_states() super().__init__() # FSMBase.__init__() requires states From a2d1531ea49d1519848be27f846f25cb367dca42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 27 Jul 2018 15:03:16 +0200 Subject: [PATCH 14/48] Fix sphinx docs broken after dependency hell --- docs/source/foundations/eventhandlers.rst | 7 +++++-- docs/source/foundations/fsms.rst | 5 ++++- docs/source/networking/networking.rst | 7 ++++++- lib/tfw/fsm/fsm_base.py | 5 +++++ lib/tfw/fsm/linear_fsm.py | 3 +++ 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/source/foundations/eventhandlers.rst b/docs/source/foundations/eventhandlers.rst index 9b8d91c..be0a05e 100644 --- a/docs/source/foundations/eventhandlers.rst +++ b/docs/source/foundations/eventhandlers.rst @@ -3,10 +3,13 @@ Event handler base classes Subclass these to create your cusom event handlers. -.. automodule:: tfw +.. automodule:: tfw.event_handler_base .. autoclass:: EventHandlerBase :members: -.. autoclass:: TriggeredEventHandler +.. autoclass:: FSMAwareEventHandler + :members: + +.. autoclass:: BroadcastingEventHandler :members: diff --git a/docs/source/foundations/fsms.rst b/docs/source/foundations/fsms.rst index 190ee8b..ce50b31 100644 --- a/docs/source/foundations/fsms.rst +++ b/docs/source/foundations/fsms.rst @@ -3,10 +3,13 @@ FSM base classes Subclass these to create an FSM that fits your tutorial/challenge. -.. automodule:: tfw +.. automodule:: tfw.fsm .. autoclass:: FSMBase :members: .. autoclass:: LinearFSM :members: + +.. autoclass:: YamlFSM + :members: diff --git a/docs/source/networking/networking.rst b/docs/source/networking/networking.rst index 317d266..be18772 100644 --- a/docs/source/networking/networking.rst +++ b/docs/source/networking/networking.rst @@ -6,7 +6,7 @@ Networking .. autoclass:: TFWServerConnector :members: -.. automodule:: tfw.networking.event_handlers +.. automodule:: tfw.networking.event_handlers.server_connector .. autoclass:: ServerUplinkConnector :members: @@ -15,3 +15,8 @@ Networking .. autoclass:: MessageSender :members: + +.. automodule:: tfw.networking.fsm_aware + +.. autoclass:: FSMAware + :members: diff --git a/lib/tfw/fsm/fsm_base.py b/lib/tfw/fsm/fsm_base.py index 5b58ce3..d9fff29 100644 --- a/lib/tfw/fsm/fsm_base.py +++ b/lib/tfw/fsm/fsm_base.py @@ -23,6 +23,11 @@ class FSMBase(Machine, CallbackMixin): states, transitions = [], [] def __init__(self, initial=None, accepted_states=None): + """ + :param initial: which state to begin with, defaults to the last one + :param accepted_states: list of states in which the challenge should be + considered successfully completed + """ self.accepted_states = accepted_states or [self.states[-1].name] self.trigger_predicates = defaultdict(list) self.event_log = [] diff --git a/lib/tfw/fsm/linear_fsm.py b/lib/tfw/fsm/linear_fsm.py index e515d8f..5ac5eaf 100644 --- a/lib/tfw/fsm/linear_fsm.py +++ b/lib/tfw/fsm/linear_fsm.py @@ -16,6 +16,9 @@ class LinearFSM(FSMBase): (0) -- step_next --> (1) -- step_next --> (2) -- step_next --> (3) ... """ def __init__(self, number_of_steps): + """ + :param number_of_steps: how many states this FSM should have + """ self.states = [State(name=str(index)) for index in range(number_of_steps)] self.transitions = [] for state in self.states[:-1]: From 199e1a5d6eec2dfac7dac659b77e7e3c1596763f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 30 Jul 2018 09:00:19 +0200 Subject: [PATCH 15/48] Update readme with Python3 event handler info --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index d46cd4b..d1670ac 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ Our pre-made event handlers are written in Python3, but you can write event hand This makes the framework really flexible: you can demonstrate the concepts you want to in any language while using the same set of tools provided by TFW. Inside Avatao this means that any of the content teams can use the framework with ease. +To implement an event handler in Python3 you should subclass the `EventHandlerBase` or `FSMAwareEventHandler` class in `tfw.event_handler_base` (the first provides a minimal working `EventHandler`, the second allows you to execute code on FSM events). + ### FSM Another unique feature of the framework is the FSM – finite state machine – representing the state of your challenge. From d94bc37d48d4eefdceafef49187207b16d7dd813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 30 Jul 2018 09:25:19 +0200 Subject: [PATCH 16/48] Add event logging to FSMAware --- lib/tfw/networking/fsm_aware.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/tfw/networking/fsm_aware.py b/lib/tfw/networking/fsm_aware.py index e666d8c..cb2b287 100644 --- a/lib/tfw/networking/fsm_aware.py +++ b/lib/tfw/networking/fsm_aware.py @@ -16,7 +16,7 @@ class FSMAware: def __init__(self): self.fsm_state = None self.fsm_in_accepted_state = False - self.fsm_last_update = None + self.fsm_event_log = [] self._auth_key = KeyManager().auth_key def update_fsm_data(self, message): @@ -27,12 +27,13 @@ class FSMAware: def _handle_fsm_update(self, message): try: - new_state = message['data']['current_state'] + update_data = message['data'] + new_state = update_data['current_state'] if self.fsm_state != new_state: - self.handle_fsm_step(**(message['data'])) + self.handle_fsm_step(**update_data) self.fsm_state = new_state - self.fsm_in_accepted_state = message['data']['in_accepted_state'] - self.fsm_last_update = message['data'] + self.fsm_in_accepted_state = update_data['in_accepted_state'] + self.fsm_event_log.append(update_data) except KeyError: LOG.error('Invalid fsm_update message received!') From a04b078513dbcb09083f9275dab852158e85f7bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 30 Jul 2018 09:25:32 +0200 Subject: [PATCH 17/48] Implement message sequence numbers in ZMQWSProxy --- lib/tfw/networking/server/zmq_websocket_proxy.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/tfw/networking/server/zmq_websocket_proxy.py b/lib/tfw/networking/server/zmq_websocket_proxy.py index 6df7c9a..5ab0b10 100644 --- a/lib/tfw/networking/server/zmq_websocket_proxy.py +++ b/lib/tfw/networking/server/zmq_websocket_proxy.py @@ -14,6 +14,7 @@ LOG = logging.getLogger(__name__) class ZMQWebSocketProxy(WebSocketHandler): # pylint: disable=abstract-method instances = set() + sequence_number = 0 def initialize(self, **kwargs): # pylint: disable=arguments-differ self._event_handler_connector = kwargs['event_handler_connector'] @@ -59,14 +60,20 @@ class ZMQWebSocketProxy(WebSocketHandler): """ Invoked on ZMQ messages from event handlers. """ + self.sequence_message(message) LOG.debug('Received on pull socket: %s', message) self.proxy_eventhandler_to_websocket(message) + def sequence_message(self, message): + self.sequence_number += 1 + message['seq'] = self.sequence_number + def on_message(self, message): """ Invoked on WS messages from frontend. """ message = json.loads(message) + self.sequence_message(message) LOG.debug('Received on WebSocket: %s', message) self.proxy_websocket_to_eventhandler(message) From 3bc30ab503884cbfcafc616688d0758f0430253a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 30 Jul 2018 09:34:13 +0200 Subject: [PATCH 18/48] Add info on signature and seq keys to readme --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d1670ac..6bdba8f 100644 --- a/README.md +++ b/README.md @@ -76,20 +76,24 @@ The TFW message format: ```text { - "key: "some identifier used for addressing", + "key: ...some identifier used for addressing..., "data": { ... JSON object carrying anything, preferably cats ... }, - "trigger": "FSM action" + "trigger": ...FSM action..., + "signature": ...HMAC signature for authenticated messages..., + "seq": ...sequence number... } ``` - The `key` field is used by TFW for addressing and every message must have one (it can be an empty string though) - The `data` object can contain anything you might want to send - The `trigger` key is an optional field that triggers an FSM action with that name from the current state (whatever that might be) +- The `signature` field is present on authenticated messages (such as `fsm_update`s) +- The `seq` key is a counter incremented with each proxied message in the TFW server To mirror messages back to their sources you can use a special messaging format, in which the message to be mirrored is enveloped inside the `data` field of the outer message: From b6d72812c47e041a9d8e7a49c5fd2fcaa3ee836b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 30 Jul 2018 15:16:03 +0200 Subject: [PATCH 19/48] Refactor ZMQWSProxy filter and callback initialization --- lib/tfw/networking/server/tfw_server.py | 17 ++++++++++------- .../networking/server/zmq_websocket_proxy.py | 19 ++++++++++--------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/tfw/networking/server/tfw_server.py b/lib/tfw/networking/server/tfw_server.py index fcea107..a1a0f5f 100644 --- a/lib/tfw/networking/server/tfw_server.py +++ b/lib/tfw/networking/server/tfw_server.py @@ -32,13 +32,16 @@ class TFWServer(FSMAware): self.application = Application([( r'/ws', ZMQWebSocketProxy, { 'event_handler_connector': self._event_handler_connector, - 'message_handlers': [ - self.handle_trigger, - self.handle_recover, - self.handle_fsm_update - ], - 'frontend_message_handlers': [self.save_frontend_messages] - })]) + 'proxy_filters_and_callbacks': { + 'message_handlers': [ + self.handle_trigger, + self.handle_recover, + self.handle_fsm_update + ], + 'frontend_message_handlers': [self.save_frontend_messages] + } + } + )]) self._frontend_messages = FrontendMessageStorage() diff --git a/lib/tfw/networking/server/zmq_websocket_proxy.py b/lib/tfw/networking/server/zmq_websocket_proxy.py index 5ab0b10..fc12a6c 100644 --- a/lib/tfw/networking/server/zmq_websocket_proxy.py +++ b/lib/tfw/networking/server/zmq_websocket_proxy.py @@ -18,11 +18,7 @@ class ZMQWebSocketProxy(WebSocketHandler): def initialize(self, **kwargs): # pylint: disable=arguments-differ self._event_handler_connector = kwargs['event_handler_connector'] - - self._message_handlers = kwargs.get('message_handlers', []) - self._frontend_message_handlers = kwargs.get('frontend_message_handlers', []) - self._eventhandler_message_handlers = kwargs.get('eventhandler_message_handlers', []) - self._proxy_filters = kwargs.get('proxy_filters', []) + self._proxy_filters_and_callbacks = kwargs.get('proxy_filters_and_callbacks', {}) self.proxy_eventhandler_to_websocket = TFWProxy( self.send_eventhandler_message, @@ -36,14 +32,19 @@ class ZMQWebSocketProxy(WebSocketHandler): self.subscribe_proxy_callbacks() def subscribe_proxy_callbacks(self): + eventhandler_message_handlers = self._proxy_filters_and_callbacks.get('eventhandler_message_handlers', []) + frontend_message_handlers = self._proxy_filters_and_callbacks.get('frontend_message_handlers', []) + message_handlers = self._proxy_filters_and_callbacks.get('message_handlers', []) + proxy_filters = self._proxy_filters_and_callbacks.get('proxy_filters', []) + self.proxy_websocket_to_eventhandler.subscribe_proxy_callbacks_and_filters( - self._eventhandler_message_handlers + self._message_handlers, - self._proxy_filters + eventhandler_message_handlers + message_handlers, + proxy_filters ) self.proxy_eventhandler_to_websocket.subscribe_proxy_callbacks_and_filters( - self._frontend_message_handlers + self._message_handlers, - self._proxy_filters + frontend_message_handlers + message_handlers, + proxy_filters ) def prepare(self): From 1ad9e0d0e3505200cc0d5e1dbb759f3315eaea20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 31 Jul 2018 14:34:28 +0200 Subject: [PATCH 20/48] Add sphinx docs on rate limiting --- docs/source/index.rst | 10 ++++++++++ docs/source/utility/decorators.rst | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 docs/source/utility/decorators.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 3d674ad..28b7a81 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -36,6 +36,16 @@ These are pre-written components for you to use, such as our IDE, terminal or co components/* +Utility +------- + +These are useful decorators, mixins and helpers to make common dev tasks easier. + +.. toctree:: + :glob: + + utility/* + Indices and tables ================== diff --git a/docs/source/utility/decorators.rst b/docs/source/utility/decorators.rst new file mode 100644 index 0000000..0e16c23 --- /dev/null +++ b/docs/source/utility/decorators.rst @@ -0,0 +1,10 @@ +TFW decorators +-------------- + +.. automodule:: tfw.decorators.rate_limiter + +.. autoclass:: RateLimiter + :members: + +.. autoclass:: AsyncRateLimiter + :members: From 14a98587a4b325d71bf59d50897daac6e8c75cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 31 Jul 2018 15:18:35 +0200 Subject: [PATCH 21/48] Silence pylint false positive --- lib/tfw/decorators/rate_limiter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tfw/decorators/rate_limiter.py b/lib/tfw/decorators/rate_limiter.py index abe6453..6f666c8 100644 --- a/lib/tfw/decorators/rate_limiter.py +++ b/lib/tfw/decorators/rate_limiter.py @@ -87,6 +87,7 @@ class AsyncRateLimiter(RateLimiter): return self._ioloop_factory() def action(self, seconds_to_next_call): + # pylint: disable=method-hidden if self._last_callback: self.ioloop.remove_timeout(self._last_callback) From aa0fe5d00fdada9e0554b700f2f3e2f7627b572d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 1 Aug 2018 14:17:02 +0200 Subject: [PATCH 22/48] Improve Dockerfile --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index a264278..5a66fd1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,9 +35,9 @@ ENV PYTHONPATH="/usr/local/lib" \ TFW_NGINX_DEFAULT="/etc/nginx/sites-enabled/default" \ TFW_NGINX_COMPONENTS="/etc/nginx/components" \ TFW_LIB_DIR="/usr/local/lib" \ - TFW_TERMINADO_DIR="/tmp/terminado_server" \ TFW_FRONTEND_DIR="/srv/frontend" \ - TFW_SERVER_DIR="/srv/.tfw" \ + TFW_DIR="/.tfw" \ + TFW_SERVER_DIR="/.tfw/tfw_server" \ TFW_AUTH_KEY="/tmp/tfw-auth.key" \ TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history" \ PROMPT_COMMAND="history -a" @@ -74,4 +74,4 @@ ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn install --frozen-lockfil ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn build --no-progress || : ONBUILD RUN test -z "${NOFRONTEND}" && mv /data/dist ${TFW_FRONTEND_DIR} && rm -rf /data || : -CMD exec supervisord --nodaemon +CMD exec supervisord --nodaemon --configuration ${TFW_SUPERVISORD_CONF} From 564c97e66a3859c4f0df690bd0ef17cd809880b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 1 Aug 2018 14:17:17 +0200 Subject: [PATCH 23/48] Remove unused variable from terminado server --- lib/tfw/components/terminal_event_handler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/components/terminal_event_handler.py index 27bbce7..7fbd74b 100644 --- a/lib/tfw/components/terminal_event_handler.py +++ b/lib/tfw/components/terminal_event_handler.py @@ -26,7 +26,6 @@ class TerminalEventHandler(EventHandlerBase): :param monitor: tfw.components.HistoryMonitor instance to read command history from """ super().__init__(key) - self.working_directory = TFWENV.TERMINADO_DIR self._historymonitor = monitor bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash'] From e3b97ee190a4466ba9bea1ee5004b73d300433b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 1 Aug 2018 17:14:56 +0200 Subject: [PATCH 24/48] Fix message sequencing not being global --- lib/tfw/networking/server/zmq_websocket_proxy.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/tfw/networking/server/zmq_websocket_proxy.py b/lib/tfw/networking/server/zmq_websocket_proxy.py index fc12a6c..15826bd 100644 --- a/lib/tfw/networking/server/zmq_websocket_proxy.py +++ b/lib/tfw/networking/server/zmq_websocket_proxy.py @@ -65,9 +65,10 @@ class ZMQWebSocketProxy(WebSocketHandler): LOG.debug('Received on pull socket: %s', message) self.proxy_eventhandler_to_websocket(message) - def sequence_message(self, message): - self.sequence_number += 1 - message['seq'] = self.sequence_number + @classmethod + def sequence_message(cls, message): + cls.sequence_number += 1 + message['seq'] = cls.sequence_number def on_message(self, message): """ From 2e97d18340e2e9fec59244f707967488331116e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 1 Aug 2018 17:18:43 +0200 Subject: [PATCH 25/48] Fix SnapshotProvider failing depending on python3.7 --- lib/tfw/components/snapshot_provider.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index edee8ab..3623f0e 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -2,7 +2,7 @@ # All Rights Reserved. See LICENSE file for details. import re -from subprocess import run, CalledProcessError +from subprocess import run, CalledProcessError, PIPE from getpass import getuser from os.path import isdir from datetime import datetime @@ -87,7 +87,8 @@ class SnapshotProvider: )) def _get_stdout(self, *args, **kwargs): - kwargs['capture_output'] = True + kwargs['stdout'] = PIPE + kwargs['stderr'] = PIPE stdout_bytes = self._run(*args, **kwargs).stdout return stdout_bytes.decode().rstrip('\n') From df0e24319d7bf5903be5bbad5134dfa48c6aa653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 1 Aug 2018 17:19:31 +0200 Subject: [PATCH 26/48] Fix SnapshotProvider failing on taking_snapshot without changes --- lib/tfw/components/snapshot_provider.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 3623f0e..aa9bb1f 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -66,10 +66,14 @@ class SnapshotProvider: 'git', 'add', '-A' )) - self._run(( - 'git', 'commit', - '-m', 'Snapshot' - )) + try: + self._get_stdout(( + 'git', 'commit', + '-m', 'Snapshot' + )) + except CalledProcessError as err: + if b'nothing to commit, working tree clean' not in err.output: + raise def _check_head_not_detached(self): if self._head_detached: From 3fee8fee2014d35870e300487d5ea2ed2154bdde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 1 Aug 2018 17:24:39 +0200 Subject: [PATCH 27/48] Implement first version of DirectorySnapshottingEventHandler --- Dockerfile | 1 + lib/tfw/components/__init__.py | 1 + .../directory_snapshotting_event_handler.py | 56 +++++++++++++++++++ requirements.txt | 1 + 4 files changed, 59 insertions(+) create mode 100644 lib/tfw/components/directory_snapshotting_event_handler.py diff --git a/Dockerfile b/Dockerfile index 5a66fd1..80229c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,7 @@ ENV PYTHONPATH="/usr/local/lib" \ TFW_FRONTEND_DIR="/srv/frontend" \ TFW_DIR="/.tfw" \ TFW_SERVER_DIR="/.tfw/tfw_server" \ + TFW_SNAPSHOTS_DIR="/.tfw/snapshots" \ TFW_AUTH_KEY="/tmp/tfw-auth.key" \ TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history" \ PROMPT_COMMAND="history -a" diff --git a/lib/tfw/components/__init__.py b/lib/tfw/components/__init__.py index 4c0475c..c75365d 100644 --- a/lib/tfw/components/__init__.py +++ b/lib/tfw/components/__init__.py @@ -10,3 +10,4 @@ from .terminal_commands import TerminalCommands from .log_monitoring_event_handler import LogMonitoringEventHandler from .fsm_managing_event_handler import FSMManagingEventHandler from .snapshot_provider import SnapshotProvider +from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler diff --git a/lib/tfw/components/directory_snapshotting_event_handler.py b/lib/tfw/components/directory_snapshotting_event_handler.py new file mode 100644 index 0000000..5a02f1a --- /dev/null +++ b/lib/tfw/components/directory_snapshotting_event_handler.py @@ -0,0 +1,56 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from os.path import join as joinpath +from os.path import basename +from os import makedirs +from uuid import uuid4 + +from dateutil import parser as dateparser + +from tfw.event_handler_base import EventHandlerBase +from tfw.components.snapshot_provider import SnapshotProvider +from tfw.config import TFWENV +from tfw.config.logs import logging + +LOG = logging.getLogger(__name__) + + +class DirectorySnapshottingEventHandler(EventHandlerBase): + def __init__(self, key, directories): + super().__init__(key) + self.snapshot_providers = {} + self.init_snapshot_providers(directories) + + self.command_handlers = { + 'take_snapshot': self.handle_take_snapshot, + 'restore_snapshot': self.handle_restore_snapshot + } + + def init_snapshot_providers(self, directories): + for directory in directories: + git_dir = joinpath( + TFWENV.SNAPSHOTS_DIR, + f'{basename(directory)}-{str(uuid4())}' + ) + makedirs(git_dir, exist_ok=True) + self.snapshot_providers[directory] = SnapshotProvider(directory, git_dir) + + def handle_event(self, message): + try: + data = message['data'] + message['data'] = self.command_handlers[data['command']](data) + return message + except KeyError: + LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) + + def handle_take_snapshot(self, data): + for provider in self.snapshot_providers.values(): + provider.take_snapshot() + return data + + def handle_restore_snapshot(self, data): + date = dateparser.parse(data['value']) + for provider in self.snapshot_providers.values(): + provider.restore_snapshot(date) + return data diff --git a/requirements.txt b/requirements.txt index 2f133de..8b061c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ watchdog==0.8.3 PyYAML==3.12 Jinja2==2.10 cryptography==2.2.2 +python-dateutil==2.7.3 From 3d2e3e7db39a2a197194d27dff0a7894e1f3cf28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 3 Aug 2018 11:39:55 +0200 Subject: [PATCH 28/48] Fix python3.7 incompatibilities in SnapshotProvider --- lib/tfw/components/snapshot_provider.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index aa9bb1f..f0cd6ba 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -8,6 +8,8 @@ from os.path import isdir from datetime import datetime from uuid import uuid4 +from dateutil import parser as dateparser + class SnapshotProvider: def __init__(self, directory, git_dir): @@ -109,7 +111,7 @@ class SnapshotProvider: self._snapshot() def _checkout_new_branch_from_head(self): - branch_name = uuid4() + branch_name = str(uuid4()) self._run(( 'git', 'branch', branch_name @@ -174,7 +176,7 @@ class SnapshotProvider: commit_hash, timestamp = line.split('@') commits.append({ 'hash': commit_hash, - 'timestamp': datetime.fromisoformat(timestamp) + 'timestamp': dateparser.parse(timestamp) }) return commits From e383be0149d025fa9966a7d2560a2d7eea72b8b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 3 Aug 2018 11:55:51 +0200 Subject: [PATCH 29/48] Fix restore_snapshot() choking on timestamps before initial commit --- lib/tfw/components/snapshot_provider.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index f0cd6ba..2c8dcee 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -129,13 +129,24 @@ class SnapshotProvider: self._checkout(commit) def _get_commit_from_timestamp(self, date): - return self._get_stdout(( + commit = self._get_stdout(( 'git', 'rev-list', '--date=iso', '-n', '1', f'--before="{date.isoformat()}"', self._last_valid_branch )) + if not commit: + commit = self._get_oldest_parent_of_head() + return commit + + def _get_oldest_parent_of_head(self): + return self._get_stdout(( + 'git', + 'rev-list', + '--max-parents=0', + 'HEAD' + )) @property def _last_valid_branch(self): From 088a1cefc55f2a18819a99c3d51b1ba5ee760b62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 3 Aug 2018 11:57:22 +0200 Subject: [PATCH 30/48] Fix initialization issues with DirectorySnapshottingEH --- .../directory_snapshotting_event_handler.py | 25 +++++++++++++++---- lib/tfw/components/snapshot_provider.py | 1 - 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/tfw/components/directory_snapshotting_event_handler.py b/lib/tfw/components/directory_snapshotting_event_handler.py index 5a02f1a..3d50315 100644 --- a/lib/tfw/components/directory_snapshotting_event_handler.py +++ b/lib/tfw/components/directory_snapshotting_event_handler.py @@ -5,6 +5,7 @@ from os.path import join as joinpath from os.path import basename from os import makedirs from uuid import uuid4 +from glob import glob from dateutil import parser as dateparser @@ -29,13 +30,21 @@ class DirectorySnapshottingEventHandler(EventHandlerBase): def init_snapshot_providers(self, directories): for directory in directories: - git_dir = joinpath( - TFWENV.SNAPSHOTS_DIR, - f'{basename(directory)}-{str(uuid4())}' - ) - makedirs(git_dir, exist_ok=True) + git_dir = self.init_git_dir(directory) self.snapshot_providers[directory] = SnapshotProvider(directory, git_dir) + @staticmethod + def init_git_dir(directory): + git_dir_prefix = joinpath( + TFWENV.SNAPSHOTS_DIR, + f'{basename(directory)}-' + ) + potential_dirs = glob(f'{git_dir_prefix}*') + + git_dir = potential_dirs[0] if potential_dirs else f'{git_dir_prefix}{str(uuid4())}' + makedirs(git_dir, exist_ok=True) + return git_dir + def handle_event(self, message): try: data = message['data'] @@ -45,12 +54,18 @@ class DirectorySnapshottingEventHandler(EventHandlerBase): LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) def handle_take_snapshot(self, data): + LOG.debug('Taking snapshots of directories %s', self.snapshot_providers.keys()) for provider in self.snapshot_providers.values(): provider.take_snapshot() return data def handle_restore_snapshot(self, data): date = dateparser.parse(data['value']) + LOG.debug( + 'Restoring snapshots (@ %s) of directories %s', + date, + self.snapshot_providers.keys() + ) for provider in self.snapshot_providers.values(): provider.restore_snapshot(date) return data diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 2c8dcee..11c39c6 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -5,7 +5,6 @@ import re from subprocess import run, CalledProcessError, PIPE from getpass import getuser from os.path import isdir -from datetime import datetime from uuid import uuid4 from dateutil import parser as dateparser From 7572699e55653b2f846bc0311ecbec51cd4a9532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 3 Aug 2018 13:40:34 +0200 Subject: [PATCH 31/48] Start working on something better than == for history checks --- lib/tfw/components/commands_equal.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 lib/tfw/components/commands_equal.py diff --git a/lib/tfw/components/commands_equal.py b/lib/tfw/components/commands_equal.py new file mode 100644 index 0000000..713d227 --- /dev/null +++ b/lib/tfw/components/commands_equal.py @@ -0,0 +1,19 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from shlex import split + + +class CommandsEqual: + def __init__(self, command_1, command_2): + self.command_1 = command_1 + self.command_2 = command_2 + + def __bool__(self): + parts_1 = split(self.command_1) + parts_2 = split(self.command_2) + return ( + parts_1[0] == parts_2[0] + and set(parts_1) == set(parts_2) + ) + From b6d8f7913f80331ad2ad35bc9528be693175cf24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 3 Aug 2018 15:01:44 +0200 Subject: [PATCH 32/48] Improve CommandsEqual with fuzzy logic --- lib/tfw/components/commands_equal.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/lib/tfw/components/commands_equal.py b/lib/tfw/components/commands_equal.py index 713d227..64ea20c 100644 --- a/lib/tfw/components/commands_equal.py +++ b/lib/tfw/components/commands_equal.py @@ -3,17 +3,23 @@ from shlex import split +from tfw.decorators.lazy_property import lazy_property + class CommandsEqual: - def __init__(self, command_1, command_2): - self.command_1 = command_1 - self.command_2 = command_2 + def __init__(self, command_1, command_2, fuzzyness=1): + self.command_1 = split(command_1) + self.command_2 = split(command_2) + self.fuzzyness = fuzzyness def __bool__(self): - parts_1 = split(self.command_1) - parts_2 = split(self.command_2) - return ( - parts_1[0] == parts_2[0] - and set(parts_1) == set(parts_2) - ) + return self.similarity >= self.fuzzyness + @lazy_property + def similarity(self): + parts_1 = set(self.command_1) + parts_2 = set(self.command_2) + + difference = parts_1 - parts_2 + deviance = len(difference) / len(max(parts_1, parts_2)) + return 1 - deviance From 8454236bc82610536d5f3edab0c77b4acd08b9f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 3 Aug 2018 15:13:02 +0200 Subject: [PATCH 33/48] Implement must_begin_similarly constraint CommandsEqual --- lib/tfw/components/commands_equal.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/tfw/components/commands_equal.py b/lib/tfw/components/commands_equal.py index 64ea20c..d58c064 100644 --- a/lib/tfw/components/commands_equal.py +++ b/lib/tfw/components/commands_equal.py @@ -7,14 +7,22 @@ from tfw.decorators.lazy_property import lazy_property class CommandsEqual: - def __init__(self, command_1, command_2, fuzzyness=1): + def __init__(self, command_1, command_2, fuzzyness=1, must_begin_similarly=True): self.command_1 = split(command_1) self.command_2 = split(command_2) self.fuzzyness = fuzzyness + self.must_begin_similarly = must_begin_similarly def __bool__(self): + if self.must_begin_similarly and not self.beginnings_are_equal: + return False + return self.similarity >= self.fuzzyness + @lazy_property + def beginnings_are_equal(self): + return self.command_1[0] == self.command_2[0] + @lazy_property def similarity(self): parts_1 = set(self.command_1) From 4f881a0ea017ad6606b1db7a032eec9166a60fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 3 Aug 2018 16:07:12 +0200 Subject: [PATCH 34/48] Implement must_contain_patterns CommandsEqual --- lib/tfw/components/commands_equal.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/tfw/components/commands_equal.py b/lib/tfw/components/commands_equal.py index d58c064..22dbe86 100644 --- a/lib/tfw/components/commands_equal.py +++ b/lib/tfw/components/commands_equal.py @@ -2,27 +2,46 @@ # All Rights Reserved. See LICENSE file for details. from shlex import split +from re import search from tfw.decorators.lazy_property import lazy_property class CommandsEqual: - def __init__(self, command_1, command_2, fuzzyness=1, must_begin_similarly=True): + def __init__(self, command_1, command_2, fuzzyness=1, must_begin_similarly=True, must_contain_patterns=None): self.command_1 = split(command_1) self.command_2 = split(command_2) self.fuzzyness = fuzzyness self.must_begin_similarly = must_begin_similarly + self.must_contain_patterns = must_contain_patterns def __bool__(self): if self.must_begin_similarly and not self.beginnings_are_equal: return False + if self.must_contain_patterns is not None and not self.commands_contain_necessary_parts: + return False + return self.similarity >= self.fuzzyness @lazy_property def beginnings_are_equal(self): return self.command_1[0] == self.command_2[0] + @lazy_property + def commands_contain_necessary_parts(self): + return all(( + self.contains_neccessary_parts(self.command_1), + self.contains_neccessary_parts(self.command_2) + )) + + def contains_neccessary_parts(self, command): + command = ' '.join(command) + for pattern in self.must_contain_patterns: + if not search(pattern, command): + return False + return True + @lazy_property def similarity(self): parts_1 = set(self.command_1) From 16c936b2cda63d0b4274dd9a2d532b193cc0980c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 4 Aug 2018 21:12:06 +0200 Subject: [PATCH 35/48] Add exclude_patterns support for CommandsEqual --- lib/tfw/components/commands_equal.py | 44 ++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/lib/tfw/components/commands_equal.py b/lib/tfw/components/commands_equal.py index 22dbe86..5115c71 100644 --- a/lib/tfw/components/commands_equal.py +++ b/lib/tfw/components/commands_equal.py @@ -8,20 +8,32 @@ from tfw.decorators.lazy_property import lazy_property class CommandsEqual: - def __init__(self, command_1, command_2, fuzzyness=1, must_begin_similarly=True, must_contain_patterns=None): + def __init__( + self, command_1, command_2, + fuzzyness=1, begin_similarly=True, + include_patterns=None, exclude_patterns=None + ): self.command_1 = split(command_1) self.command_2 = split(command_2) self.fuzzyness = fuzzyness - self.must_begin_similarly = must_begin_similarly - self.must_contain_patterns = must_contain_patterns + self.begin_similarly = begin_similarly + self.include_patterns = include_patterns + self.exclude_patterns = exclude_patterns def __bool__(self): - if self.must_begin_similarly and not self.beginnings_are_equal: - return False + if self.begin_similarly: + if not self.beginnings_are_equal: + return False - if self.must_contain_patterns is not None and not self.commands_contain_necessary_parts: - return False + if self.include_patterns is not None: + if not self.commands_contain_include_patterns: + return False + if self.exclude_patterns is not None: + if not self.commands_contain_no_exclude_patterns: + return False + + print(self.similarity) return self.similarity >= self.fuzzyness @lazy_property @@ -29,15 +41,23 @@ class CommandsEqual: return self.command_1[0] == self.command_2[0] @lazy_property - def commands_contain_necessary_parts(self): + def commands_contain_include_patterns(self): return all(( - self.contains_neccessary_parts(self.command_1), - self.contains_neccessary_parts(self.command_2) + self.contains_regex_patterns(self.command_1, self.include_patterns), + self.contains_regex_patterns(self.command_2, self.include_patterns) )) - def contains_neccessary_parts(self, command): + @lazy_property + def commands_contain_no_exclude_patterns(self): + return all(( + not self.contains_regex_patterns(self.command_1, self.exclude_patterns), + not self.contains_regex_patterns(self.command_2, self.exclude_patterns) + )) + + @staticmethod + def contains_regex_patterns(command, regex_parts): command = ' '.join(command) - for pattern in self.must_contain_patterns: + for pattern in regex_parts: if not search(pattern, command): return False return True From f6d77e1132615101837164cdc8f2067d8d9bedaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 4 Aug 2018 21:49:06 +0200 Subject: [PATCH 36/48] Add docstrings to CommandsEqual --- docs/source/components/components.rst | 7 +++++ lib/tfw/components/__init__.py | 1 + lib/tfw/components/commands_equal.py | 38 +++++++++++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/docs/source/components/components.rst b/docs/source/components/components.rst index 87dee5b..d2cef53 100644 --- a/docs/source/components/components.rst +++ b/docs/source/components/components.rst @@ -23,3 +23,10 @@ Components .. autoclass:: BashMonitor :members: + +.. autoclass:: FSMManagingEventHandler + :members: + +.. autoclass:: CommandsEqual + :members: + diff --git a/lib/tfw/components/__init__.py b/lib/tfw/components/__init__.py index c75365d..2960e38 100644 --- a/lib/tfw/components/__init__.py +++ b/lib/tfw/components/__init__.py @@ -11,3 +11,4 @@ from .log_monitoring_event_handler import LogMonitoringEventHandler from .fsm_managing_event_handler import FSMManagingEventHandler from .snapshot_provider import SnapshotProvider from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler +from .commands_equal import CommandsEqual diff --git a/lib/tfw/components/commands_equal.py b/lib/tfw/components/commands_equal.py index 5115c71..54700c2 100644 --- a/lib/tfw/components/commands_equal.py +++ b/lib/tfw/components/commands_equal.py @@ -8,11 +8,49 @@ from tfw.decorators.lazy_property import lazy_property class CommandsEqual: + """ + This class is useful for comparing executed commands with + excepted commands (i.e. when triggering a state change when + the correct command is executed). + + Note that in most cases you should test the changes + caused by the commands instead of just checking command history + (stuff can be done in countless ways and preparing for every + single case is impossible). This should only be used when + testing the changes would be very difficult, like when + explaining stuff with cli tools and such. + + This class implicitly converts to bool, use it like + if CommandsEqual(...): ... + + It tries detecting differing command parameter orders with similar + semantics and provides fuzzy logic options. + The rationale behind this is that a few false positives + are better than only accepting a single version of a command + (i.e. using ==). + """ def __init__( self, command_1, command_2, fuzzyness=1, begin_similarly=True, include_patterns=None, exclude_patterns=None ): + """ + :param command_1: Compared command 1 + :param command_2: Compared command 2 + :param fuzzyness: float between 0 and 1. + the percentage of arguments required to + match between commands to result in True. + i.e 1 means 100% - all arguments need to be + present in both commands, while 0.75 + would mean 75% - in case of 4 arguments + 1 could differ between the commands. + :param begin_similarly: bool, the first word of the commands + must match + :param include_patterns: list of regex patterns the commands + must include + :param exclude_patterns: list of regex patterns the commands + must exclude + """ self.command_1 = split(command_1) self.command_2 = split(command_2) self.fuzzyness = fuzzyness From 01e55778900faaff92e5dd605a0ba60fa95be619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 4 Aug 2018 23:27:18 +0200 Subject: [PATCH 37/48] Document FSMManagingEventHandler --- .../components/fsm_managing_event_handler.py | 23 +++++++++++++++++++ lib/tfw/components/ide_event_handler.py | 3 ++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/tfw/components/fsm_managing_event_handler.py b/lib/tfw/components/fsm_managing_event_handler.py index 376ce50..73ebe10 100644 --- a/lib/tfw/components/fsm_managing_event_handler.py +++ b/lib/tfw/components/fsm_managing_event_handler.py @@ -9,6 +9,20 @@ LOG = logging.getLogger(__name__) class FSMManagingEventHandler(EventHandlerBase): + """ + EventHandler responsible for managing the state machine of + the framework (TFW FSM). + + tfw.networking.TFWServer instances automatically send 'trigger' + commands to the event handler listening on the 'fsm' key, + which should be an instance of this event handler. + + This event handler accepts messages that have a + data['command'] key specifying a command to be executed. + + An 'fsm_update' message is broadcasted after every successful + command. + """ def __init__(self, key, fsm_type, require_signature=False): super().__init__(key) self.fsm = fsm_type() @@ -34,6 +48,12 @@ class FSMManagingEventHandler(EventHandlerBase): LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) def handle_trigger(self, message): + """ + Attempts to step the FSM with the supplied trigger. + + :param message: TFW message with a data field containing + the action to try triggering in data['value'] + """ trigger = message['data']['value'] if self._require_signature: if not verify_message(self.auth_key, message): @@ -44,6 +64,9 @@ class FSMManagingEventHandler(EventHandlerBase): return None def handle_update(self, message): + """ + Does nothing, but triggers an 'fsm_update' message. + """ # pylint: disable=no-self-use return message diff --git a/lib/tfw/components/ide_event_handler.py b/lib/tfw/components/ide_event_handler.py index 05905be..7e242e5 100644 --- a/lib/tfw/components/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler.py @@ -157,7 +157,8 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): """ Read the currently selected file. - :return dict: message with the contents of the file in data['content'] + :return dict: TFW message data containing key 'content' + (contents of the selected file) """ try: data['content'] = self.filemanager.file_contents From b140550686a6c6fa6cf048202440cbcdbe509883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 6 Aug 2018 13:16:20 +0200 Subject: [PATCH 38/48] Remove debugging log from CommandsEqual --- lib/tfw/components/commands_equal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/tfw/components/commands_equal.py b/lib/tfw/components/commands_equal.py index 54700c2..8530a4c 100644 --- a/lib/tfw/components/commands_equal.py +++ b/lib/tfw/components/commands_equal.py @@ -71,7 +71,6 @@ class CommandsEqual: if not self.commands_contain_no_exclude_patterns: return False - print(self.similarity) return self.similarity >= self.fuzzyness @lazy_property From 782df25bee8d2e8b929fb75924cb8ca0951749f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 6 Aug 2018 13:40:16 +0200 Subject: [PATCH 39/48] =?UTF-8?q?Fix=20broken=20init=5Fgit=5Fdir=20logic?= =?UTF-8?q?=20=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../directory_snapshotting_event_handler.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/lib/tfw/components/directory_snapshotting_event_handler.py b/lib/tfw/components/directory_snapshotting_event_handler.py index 3d50315..246bb60 100644 --- a/lib/tfw/components/directory_snapshotting_event_handler.py +++ b/lib/tfw/components/directory_snapshotting_event_handler.py @@ -4,8 +4,6 @@ from os.path import join as joinpath from os.path import basename from os import makedirs -from uuid import uuid4 -from glob import glob from dateutil import parser as dateparser @@ -29,19 +27,16 @@ class DirectorySnapshottingEventHandler(EventHandlerBase): } def init_snapshot_providers(self, directories): - for directory in directories: - git_dir = self.init_git_dir(directory) + for index, directory in enumerate(directories): + git_dir = self.init_git_dir(index, directory) self.snapshot_providers[directory] = SnapshotProvider(directory, git_dir) @staticmethod - def init_git_dir(directory): - git_dir_prefix = joinpath( + def init_git_dir(index, directory): + git_dir = joinpath( TFWENV.SNAPSHOTS_DIR, - f'{basename(directory)}-' + f'{basename(directory)}-{index}' ) - potential_dirs = glob(f'{git_dir_prefix}*') - - git_dir = potential_dirs[0] if potential_dirs else f'{git_dir_prefix}{str(uuid4())}' makedirs(git_dir, exist_ok=True) return git_dir From cbb807dfb46241fc2095b9a927a056667946e064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 6 Aug 2018 14:19:18 +0200 Subject: [PATCH 40/48] Implement restore_snapshot latest commit on branch detection --- lib/tfw/components/snapshot_provider.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 11c39c6..8fbf624 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -10,6 +10,7 @@ from uuid import uuid4 from dateutil import parser as dateparser +# TODO: gitignore class SnapshotProvider: def __init__(self, directory, git_dir): self._classname = self.__class__.__name__ @@ -125,6 +126,9 @@ class SnapshotProvider: def restore_snapshot(self, date): commit = self._get_commit_from_timestamp(date) + branch = self._last_valid_branch + if commit == self._latest_commit_on_branch(branch): + commit = branch self._checkout(commit) def _get_commit_from_timestamp(self, date): @@ -153,6 +157,14 @@ class SnapshotProvider: self.__last_valid_branch = self._branch return self.__last_valid_branch + def _latest_commit_on_branch(self, branch): + return self._get_stdout(( + 'git', 'log', + '-n', '1', + '--pretty=format:%H', + branch + )) + @property def all_timelines(self): return self._branches From 59dce4a8484ddea35311ade9f0d737c171a33a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 6 Aug 2018 14:55:58 +0200 Subject: [PATCH 41/48] Restore latest snapshot if no date is provided --- .../components/directory_snapshotting_event_handler.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/tfw/components/directory_snapshotting_event_handler.py b/lib/tfw/components/directory_snapshotting_event_handler.py index 246bb60..55fcde8 100644 --- a/lib/tfw/components/directory_snapshotting_event_handler.py +++ b/lib/tfw/components/directory_snapshotting_event_handler.py @@ -4,6 +4,7 @@ from os.path import join as joinpath from os.path import basename from os import makedirs +from datetime import datetime from dateutil import parser as dateparser @@ -55,7 +56,12 @@ class DirectorySnapshottingEventHandler(EventHandlerBase): return data def handle_restore_snapshot(self, data): - date = dateparser.parse(data['value']) + date = dateparser.parse( + data.get( + 'value', + datetime.now().isoformat() + ) + ) LOG.debug( 'Restoring snapshots (@ %s) of directories %s', date, From b7ed4c3d0fcc157c4f43dbaae7d6230149e47018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 6 Aug 2018 15:42:51 +0200 Subject: [PATCH 42/48] Implement gitignore functionality in SnapshotProvider --- lib/tfw/components/snapshot_provider.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 8fbf624..8bd17bf 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -5,14 +5,14 @@ import re from subprocess import run, CalledProcessError, PIPE from getpass import getuser from os.path import isdir +from os.path import join as joinpath from uuid import uuid4 from dateutil import parser as dateparser -# TODO: gitignore class SnapshotProvider: - def __init__(self, directory, git_dir): + def __init__(self, directory, git_dir, exclude_unix_patterns=None): self._classname = self.__class__.__name__ author = f'{getuser()} via TFW {self._classname}' self.gitenv = { @@ -27,6 +27,8 @@ class SnapshotProvider: self._init_repo() self.__last_valid_branch = self._branch + if exclude_unix_patterns: + self.exclude = exclude_unix_patterns def _init_repo(self): self._check_environment() @@ -105,6 +107,24 @@ class SnapshotProvider: kwargs['env'] = self.gitenv return run(*args, **kwargs) + @property + def exclude(self): + with open(self._exclude_path, 'r') as ofile: + return ofile.read() + + @exclude.setter + def exclude(self, exclude_patterns): + with open(self._exclude_path, 'w') as ifile: + ifile.write('\n'.join(exclude_patterns)) + + @property + def _exclude_path(self): + return joinpath( + self.gitenv['GIT_DIR'], + 'info', + 'exclude' + ) + def take_snapshot(self): if self._head_detached: self._checkout_new_branch_from_head() From 44bdc965470f20a8d3d4d3e50b07ee8f6076b883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 6 Aug 2018 15:47:14 +0200 Subject: [PATCH 43/48] Support changing gitignore from event handler API --- .../directory_snapshotting_event_handler.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/tfw/components/directory_snapshotting_event_handler.py b/lib/tfw/components/directory_snapshotting_event_handler.py index 55fcde8..8b6f384 100644 --- a/lib/tfw/components/directory_snapshotting_event_handler.py +++ b/lib/tfw/components/directory_snapshotting_event_handler.py @@ -17,20 +17,26 @@ LOG = logging.getLogger(__name__) class DirectorySnapshottingEventHandler(EventHandlerBase): - def __init__(self, key, directories): + def __init__(self, key, directories, exclude_unix_patterns=None): super().__init__(key) self.snapshot_providers = {} + self._exclude_unix_patterns = exclude_unix_patterns self.init_snapshot_providers(directories) self.command_handlers = { 'take_snapshot': self.handle_take_snapshot, - 'restore_snapshot': self.handle_restore_snapshot + 'restore_snapshot': self.handle_restore_snapshot, + 'exclude': self.handle_exclude } def init_snapshot_providers(self, directories): for index, directory in enumerate(directories): git_dir = self.init_git_dir(index, directory) - self.snapshot_providers[directory] = SnapshotProvider(directory, git_dir) + self.snapshot_providers[directory] = SnapshotProvider( + directory, + git_dir, + self._exclude_unix_patterns + ) @staticmethod def init_git_dir(index, directory): @@ -70,3 +76,12 @@ class DirectorySnapshottingEventHandler(EventHandlerBase): for provider in self.snapshot_providers.values(): provider.restore_snapshot(date) return data + + def handle_exclude(self, data): + exclude_unix_patterns = data['value'] + if not isinstance(exclude_unix_patterns, list): + raise KeyError + + for provider in self.snapshot_providers.values(): + provider.exclude = exclude_unix_patterns + return data From 21f05ad850249fb3f3f68ea6cd6a78cd6312dab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 6 Aug 2018 15:52:01 +0200 Subject: [PATCH 44/48] Silence unjust pylint warning --- lib/tfw/components/commands_equal.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tfw/components/commands_equal.py b/lib/tfw/components/commands_equal.py index 8530a4c..d2d4d15 100644 --- a/lib/tfw/components/commands_equal.py +++ b/lib/tfw/components/commands_equal.py @@ -8,6 +8,7 @@ from tfw.decorators.lazy_property import lazy_property class CommandsEqual: + # pylint: disable=too-many-arguments """ This class is useful for comparing executed commands with excepted commands (i.e. when triggering a state change when From 48c3df621b00878c65838438eedb4cd15df2246d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 7 Aug 2018 17:28:56 +0200 Subject: [PATCH 45/48] Add readme info on delaying histfile appending --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 6bdba8f..49574fc 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,13 @@ Overwriting the current list of excluded file patterns is possible with this mes ### TerminalEventHandler +By default callbacks on terminal history are invoked *as soon as* a command starts to execute in the terminal (they do not wait for the started command to finish, the callback may even run in paralell with the command). + +If you want to wait for them and invoke your callbacks *after* the command has finished, please set the `TFW_DELAY_HISTAPPEND` envvar to `1`. +Practically this can be done by appending an `export` to the user's `.bashrc` file from your `Dockerfile`, like so: + +`RUN echo "export TFW_DELAY_HISTAPPEND=1" >> /home/${AVATAO_USER}/.bashrc` + Writing to the terminal: ``` { From 806623c80dd11438ad916b6bba47d0e84f6c218e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 8 Aug 2018 19:27:58 +0200 Subject: [PATCH 46/48] Extend API documentation --- README.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/README.md b/README.md index 49574fc..be9b156 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,8 @@ APIs exposed by our pre-witten event handlers are documented here. ### IdeEventHandler +This event handler is responsible for reading and writing files shown in the frontend code editor. + You can read the content of the currently selected file like so: ``` { @@ -184,6 +186,8 @@ Overwriting the current list of excluded file patterns is possible with this mes ### TerminalEventHandler +Event handler responsible for running a backend for `xterm.js` to connect to (frontend terminal backend). + By default callbacks on terminal history are invoked *as soon as* a command starts to execute in the terminal (they do not wait for the started command to finish, the callback may even run in paralell with the command). If you want to wait for them and invoke your callbacks *after* the command has finished, please set the `TFW_DELAY_HISTAPPEND` envvar to `1`. @@ -217,6 +221,8 @@ You can read terminal command history like so: ### ProcessManagingEventHandler +This event handler is responsible for managing processes controlled by supervisord. + Starting, stopping and restarting supervisor processes can be done using similar messages (where `command` is `start`, `stop` or `restart`): ``` { @@ -231,6 +237,8 @@ Starting, stopping and restarting supervisor processes can be done using similar ### LogMonitoringEventHandler +Event handler emitting real time logs (`stdout` and `stderr`) from supervisord processes. + To change which supervisor process is monitored use this message: ``` { @@ -257,6 +265,8 @@ To set the tail length of logs (the monitor will send back the last `value` char ### FSMManagingEventHandler +This event handler controls the TFW finite state machine (FSM). + To attempt executing a trigger on the FSM use (this will also generate an FSM update message): ``` { @@ -292,3 +302,41 @@ This event handler broadcasts FSM update messages after handling commands in the } ``` +### DirectorySnapshottingEventHandler + +Event handler capable of taking and restoring snapshots of directories (saving and restoring directory contens). + +You can take a snapshot of the directories with the following message: +``` +{ + "key": "snapshot", + "data" : + { + "command": "take_snapshot" + } +} +``` + +To restore the state of the files in the directories use: +``` +{ + "key": "snapshot", + "data" : + { + "command": "restore_snapshot", + "value": ...date string (can parse ISO 8601, unix timestamp, etc.)... + } +} +``` + +It is also possible to exclude files that match given patterns (formatted like lines in `.gitignore` files): +``` +{ + "key": "snapshot", + "data" : + { + "command": "exclude", + "value": ...list of patterns to exclude from snapshots... + } +} +``` From 031400c0c46448835bcfda0bb3e7d4941b099781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 22 Aug 2018 14:09:53 +0200 Subject: [PATCH 47/48] Update pip packages --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8b061c5..c2ab3ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -tornado==5.0 -pyzmq==17.0.0 -transitions==0.6.4 +tornado==5.1 +pyzmq==17.1.2 +transitions==0.6.6 terminado==0.8.1 watchdog==0.8.3 -PyYAML==3.12 +PyYAML==3.13 Jinja2==2.10 -cryptography==2.2.2 +cryptography==2.3.1 python-dateutil==2.7.3 From 1d969f0967ca639ba7d66cae83326da1a850cdaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 9 Oct 2018 11:25:11 +0200 Subject: [PATCH 48/48] =?UTF-8?q?Implement=20lazy=5Ffactory=20=E2=9C=A8?= =?UTF-8?q?=F0=9F=8D=B0=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/tfw/decorators/lazy_property.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/tfw/decorators/lazy_property.py b/lib/tfw/decorators/lazy_property.py index c7e00c6..14ad788 100644 --- a/lib/tfw/decorators/lazy_property.py +++ b/lib/tfw/decorators/lazy_property.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from functools import update_wrapper +from functools import update_wrapper, wraps class lazy_property: @@ -19,3 +19,12 @@ class lazy_property: value = self.func(instance) setattr(instance, self.func.__name__, value) return value + + +def lazy_factory(fun): + class wrapper: + @wraps(fun) + @lazy_property + def instance(self): # pylint: disable=no-self-use + return fun() + return wrapper()