From f6cc7ae495cd247778ab3861c3d3b64cdffd8fc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 12 Jul 2018 15:55:26 +0200 Subject: [PATCH 01/43] Refactor ZMQWSProxy.__init__ --- .../networking/server/zmq_websocket_proxy.py | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/lib/tfw/networking/server/zmq_websocket_proxy.py b/lib/tfw/networking/server/zmq_websocket_proxy.py index 85d6c60..3fc5bff 100644 --- a/lib/tfw/networking/server/zmq_websocket_proxy.py +++ b/lib/tfw/networking/server/zmq_websocket_proxy.py @@ -16,8 +16,14 @@ class ZMQWebSocketProxy(WebSocketHandler): def initialize(self, **kwargs): # pylint: disable=arguments-differ self._event_handler_connector = kwargs['event_handler_connector'] - self._message_handlers = kwargs.get('message_handlers', []) - self._proxy_filters = kwargs.get('proxy_filters', []) + + self.init_callback_lists_from( + kwargs, + [ + 'message_handlers', + 'proxy_filters' + ] + ) self.proxy_eventhandler_to_websocket = TFWProxy( self.send_eventhandler_message, @@ -28,7 +34,21 @@ class ZMQWebSocketProxy(WebSocketHandler): self.send_eventhandler_message ) - proxies = (self.proxy_eventhandler_to_websocket, self.proxy_websocket_to_eventhandler) + self.subscribe_proxy_callbacks() + + def init_callback_lists_from(self, init_kwargs, callback_lists): + for callback_list in callback_lists: + setattr( + self, + f'_{callback_list}', + init_kwargs.get(callback_list, []) + ) + + def subscribe_proxy_callbacks(self): + proxies = ( + self.proxy_eventhandler_to_websocket, + self.proxy_websocket_to_eventhandler + ) for proxy in proxies: proxy.proxy_filters.subscribe_callbacks(*self._proxy_filters) proxy.proxy_callbacks.subscribe_callbacks(*self._message_handlers) From d2084b2e5137225b2ba94f602b4c89a46568e484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 13 Jul 2018 14:08:30 +0200 Subject: [PATCH 02/43] Add frontend/eventhandler message callback capabilities to ZMQWSProxy --- .../networking/server/zmq_websocket_proxy.py | 37 ++++++++----------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/lib/tfw/networking/server/zmq_websocket_proxy.py b/lib/tfw/networking/server/zmq_websocket_proxy.py index 3fc5bff..7a38e4a 100644 --- a/lib/tfw/networking/server/zmq_websocket_proxy.py +++ b/lib/tfw/networking/server/zmq_websocket_proxy.py @@ -17,13 +17,10 @@ class ZMQWebSocketProxy(WebSocketHandler): def initialize(self, **kwargs): # pylint: disable=arguments-differ self._event_handler_connector = kwargs['event_handler_connector'] - self.init_callback_lists_from( - kwargs, - [ - 'message_handlers', - 'proxy_filters' - ] - ) + self._message_handlers = kwargs.get('message_handlers', []) + self._frontend_message_handlers = kwargs.get('frontend_message_handlers', []) + self._eventhandler_message_handlers = kwargs.get('eventhandler_message_handlers', []) + self._proxy_filters = kwargs.get('proxy_filters', []) self.proxy_eventhandler_to_websocket = TFWProxy( self.send_eventhandler_message, @@ -36,22 +33,16 @@ class ZMQWebSocketProxy(WebSocketHandler): self.subscribe_proxy_callbacks() - def init_callback_lists_from(self, init_kwargs, callback_lists): - for callback_list in callback_lists: - setattr( - self, - f'_{callback_list}', - init_kwargs.get(callback_list, []) - ) - def subscribe_proxy_callbacks(self): - proxies = ( - self.proxy_eventhandler_to_websocket, - self.proxy_websocket_to_eventhandler + self.proxy_websocket_to_eventhandler.subscribe_proxy_callbacks_and_filters( + self._eventhandler_message_handlers + self._message_handlers, + self._proxy_filters + ) + + self.proxy_eventhandler_to_websocket.subscribe_proxy_callbacks_and_filters( + self._frontend_message_handlers + self._message_handlers, + self._proxy_filters ) - for proxy in proxies: - proxy.proxy_filters.subscribe_callbacks(*self._proxy_filters) - proxy.proxy_callbacks.subscribe_callbacks(*self._message_handlers) def prepare(self): ZMQWebSocketProxy.instances.add(self) @@ -139,3 +130,7 @@ class TFWProxy: LOG.debug('Broadcasting message: %s', message) self.to_source(message) self.to_destination(message) + + def subscribe_proxy_callbacks_and_filters(self, proxy_callbacks, proxy_filters): + self.proxy_callbacks.subscribe_callbacks(*proxy_callbacks) + self.proxy_filters.subscribe_callbacks(*proxy_filters) From 9c20e94dd83c0f926c3dc36c17a6a2744b9779f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 13 Jul 2018 14:09:13 +0200 Subject: [PATCH 03/43] Implement generating single messages from a queue --- lib/tfw/networking/message_sender.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/tfw/networking/message_sender.py b/lib/tfw/networking/message_sender.py index 58856b1..2b946d7 100644 --- a/lib/tfw/networking/message_sender.py +++ b/lib/tfw/networking/message_sender.py @@ -44,3 +44,11 @@ class MessageSender: 'key': self.queue_key, 'data': data }) + + @staticmethod + def generate_messages_from_queue(queue_messsge): + for message in queue_messsge['data']['messages']: + yield { + 'key': 'message', + 'data': message + } From 688c615fd72b0a1aaf3e1d402a354eaac523d1cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 13 Jul 2018 15:24:45 +0200 Subject: [PATCH 04/43] Implement frontend state recovery backend (partial) --- lib/tfw/networking/server/tfw_server.py | 46 ++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/lib/tfw/networking/server/tfw_server.py b/lib/tfw/networking/server/tfw_server.py index bdc5179..067bb6c 100644 --- a/lib/tfw/networking/server/tfw_server.py +++ b/lib/tfw/networking/server/tfw_server.py @@ -1,10 +1,13 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +from abc import ABC, abstractmethod + from tornado.web import Application from tfw.networking.event_handlers import ServerUplinkConnector from tfw.networking.server import EventHandlerConnector +from tfw.networking import MessageSender from tfw.config.logs import logging from .zmq_websocket_proxy import ZMQWebSocketProxy @@ -24,10 +27,13 @@ class TFWServer: self.application = Application([( r'/ws', ZMQWebSocketProxy,{ 'event_handler_connector': self._event_handler_connector, - 'message_handlers': [self.handle_trigger] + 'message_handlers': [self.handle_trigger, self.handle_recover], + 'frontend_message_handlers': [self.save_frontend_messages] })] ) + self._frontend_messages = FrontendMessageStorage() + def handle_trigger(self, message): if 'trigger' in message: LOG.debug('Executing handler for trigger "%s"', message.get('trigger', '')) @@ -39,5 +45,43 @@ class TFWServer: } }) + def handle_recover(self, message): + if message['key'] == 'recover': + self._frontend_messages.replay_messages(self._uplink_connector) + + def save_frontend_messages(self, message): + self._frontend_messages.save_message(message) + def listen(self, port): self.application.listen(port) + + +class MessageStorage(ABC): + def __init__(self): + self.saved_messages = [] + + def save_message(self, message): + if self.filter_message(message): + self.saved_messages.extend(self.transform_message(message)) + + @abstractmethod + def filter_message(self, message): + raise NotImplementedError + + def transform_message(self, message): + yield message + + +class FrontendMessageStorage(MessageStorage): + def filter_message(self, message): + return message['key'] in ('message', 'dashboard', 'queueMessages') + + def transform_message(self, message): + if message['key'] == 'queueMessages': + yield from MessageSender.generate_messages_from_queue(message) + else: + yield message + + def replay_messages(self, connector): + for message in self.saved_messages: + connector.send(message) From cfa8fc5ab6eba6154f76712be709af64cde7fe77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 13 Jul 2018 15:42:18 +0200 Subject: [PATCH 05/43] Remove deprecated 'reset' key. Sad to see you go :( --- lib/tfw/event_handler_base.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base.py index 81b1ead..511af7c 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base.py @@ -21,7 +21,7 @@ class EventHandlerBase(ABC): def __init__(self, key): self.server_connector = ServerConnector() self.key = key - self.subscribe(self.key, 'reset') + self.subscribe(self.key) self.server_connector.register_callback(self.event_handler_callback) def event_handler_callback(self, message): @@ -57,9 +57,7 @@ class EventHandlerBase(ABC): :param message: the message received :returns: the message to send back """ - if message['key'] != 'reset': - return self.handle_event(message) - return self.handle_reset(message) + return self.handle_event(message) @abstractmethod def handle_event(self, message): @@ -71,16 +69,6 @@ class EventHandlerBase(ABC): """ raise NotImplementedError - def handle_reset(self, message): - # pylint: disable=unused-argument,no-self-use - """ - Usually 'reset' events receive some sort of special treatment. - - :param message: the message received - :returns: the message to send back - """ - return None - def subscribe(self, *keys): """ Subscribe this EventHandler to receive events for given keys. From d98ce5865162f69f44588df6483b986514c6ecc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 14 Jul 2018 20:06:34 +0200 Subject: [PATCH 06/43] Add proper multikey support for EventHandlers with compatibility --- .../components/process_managing_event_handler.py | 1 - lib/tfw/event_handler_base.py | 15 ++++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/tfw/components/process_managing_event_handler.py b/lib/tfw/components/process_managing_event_handler.py index 3ccf6b1..040995e 100644 --- a/lib/tfw/components/process_managing_event_handler.py +++ b/lib/tfw/components/process_managing_event_handler.py @@ -38,7 +38,6 @@ class ProcessManagingEventHandler(EventHandlerBase): """ def __init__(self, key, dirmonitor=None, log_tail=0): super().__init__(key) - self.key = key self.monitor = dirmonitor self.processmanager = ProcessManager() self.log_tail = log_tail diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base.py index 511af7c..41f6105 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base.py @@ -20,10 +20,17 @@ class EventHandlerBase(ABC): """ def __init__(self, key): self.server_connector = ServerConnector() - self.key = key - self.subscribe(self.key) + self.keys = [key] + self.subscribe(*self.keys) self.server_connector.register_callback(self.event_handler_callback) + @property + def key(self): + """ + Returns the oldest key this EventHandler was subscribed to. + """ + return self.keys[0] + def event_handler_callback(self, message): """ Callback that is invoked when receiving a message. @@ -48,7 +55,7 @@ class EventHandlerBase(ABC): subscribed to 'fsm' will receive 'fsm_update' messages as well. """ - return self.key == message['key'] + return message['key'] in self.keys def dispatch_handling(self, message): """ @@ -80,6 +87,7 @@ class EventHandlerBase(ABC): """ for key in keys: self.server_connector.subscribe(key) + self.keys.append(key) def unsubscribe(self, *keys): """ @@ -89,6 +97,7 @@ class EventHandlerBase(ABC): """ for key in keys: self.server_connector.unsubscribe(key) + self.keys.remove(key) def cleanup(self): """ From 77f14df17bdecdda088bb63f35743547095887df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 14 Jul 2018 20:44:31 +0200 Subject: [PATCH 07/43] Implement trigger history in FSMBase --- lib/tfw/fsm_base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/tfw/fsm_base.py b/lib/tfw/fsm_base.py index 28b5ef6..2de2b43 100644 --- a/lib/tfw/fsm_base.py +++ b/lib/tfw/fsm_base.py @@ -24,6 +24,7 @@ class FSMBase(Machine, CallbackMixin): def __init__(self, initial=None, accepted_states=None): self.accepted_states = accepted_states or [self.states[-1]] self.trigger_predicates = defaultdict(list) + self.trigger_history = [] Machine.__init__( self, @@ -61,5 +62,6 @@ class FSMBase(Machine, CallbackMixin): if all(predicate_results): try: self.trigger(trigger) + self.trigger_history.append(trigger) except (AttributeError, MachineError): LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger) From 4236a17b692971d54a61a40b98d860752a39650c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 14 Jul 2018 20:45:26 +0200 Subject: [PATCH 08/43] Include last trigger in fsm_update messages --- lib/tfw/components/fsm_managing_event_handler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/tfw/components/fsm_managing_event_handler.py b/lib/tfw/components/fsm_managing_event_handler.py index f84186f..1068bbd 100644 --- a/lib/tfw/components/fsm_managing_event_handler.py +++ b/lib/tfw/components/fsm_managing_event_handler.py @@ -28,7 +28,8 @@ class FSMManagingEventHandler(EventHandlerBase): LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) def handle_trigger(self, data): - self.fsm.step(data['value']) + trigger = data['value'] + self.fsm.step(trigger) return data def handle_update(self, data): @@ -51,7 +52,9 @@ class FSMUpdater: {'trigger': trigger} for trigger in self.fsm.get_triggers(self.fsm.state) ] + last_trigger = self.fsm.trigger_history[-1] if self.fsm.trigger_history else None return { 'current_state': state, - 'valid_transitions': valid_transitions + 'valid_transitions': valid_transitions, + 'last_trigger': last_trigger } From 646a671ff309a0b1b53d402586eb3aa09f9db2a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 14 Jul 2018 20:45:56 +0200 Subject: [PATCH 09/43] Add FSMAwareEventHandler which keeps track of FSM state --- lib/tfw/__init__.py | 2 +- lib/tfw/event_handler_base.py | 32 +++++++++++++++++++++++++------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/lib/tfw/__init__.py b/lib/tfw/__init__.py index 53e80d7..04d934d 100644 --- a/lib/tfw/__init__.py +++ b/lib/tfw/__init__.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from .event_handler_base import EventHandlerBase, TriggeredEventHandler, BroadcastingEventHandler +from .event_handler_base import EventHandlerBase, FSMAwareEventHandler, BroadcastingEventHandler from .fsm_base import FSMBase from .linear_fsm import LinearFSM from .yaml_fsm import YamlFSM diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base.py index 41f6105..f6a094a 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base.py @@ -107,20 +107,38 @@ class EventHandlerBase(ABC): pass -class TriggeredEventHandler(EventHandlerBase, ABC): +class FSMAwareEventHandler(EventHandlerBase, ABC): # pylint: disable=abstract-method """ - Abstract base class for EventHandlers which are only triggered in case - TFWServer has successfully triggered an FSM step defined in __init__. + Abstract base class for EventHandlers which automatically + keep track of the state of the TFW FSM. """ - def __init__(self, key, trigger): + def __init__(self, key): super().__init__(key) - self.trigger = trigger + self.subscribe('fsm_update') + self.fsm_state = None def dispatch_handling(self, message): - if message.get('trigger') == self.trigger: + if message['key'] == 'fsm_update': + self._handle_fsm_update(message) + else: return super().dispatch_handling(message) - return None + + def _handle_fsm_update(self, message): + try: + new_state = message['data']['current_state'] + trigger = message['data']['last_trigger'] + if self.fsm_state != new_state: + self.handle_fsm_step(self.fsm_state, new_state, trigger) + self.fsm_state = new_state + except KeyError: + LOG.error('Invalid fsm_update message received!') + + def handle_fsm_step(self, from_state, to_state, trigger): + """ + Called in case the TFW FSM has stepped. + """ + pass class BroadcastingEventHandler(EventHandlerBase, ABC): From 16d98c75ca40a539b354f5c0edfe01f813b6897b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sun, 15 Jul 2018 17:26:00 +0200 Subject: [PATCH 10/43] Add message_bytes() function to serialization module --- lib/tfw/networking/__init__.py | 4 ++-- lib/tfw/networking/serialization.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/tfw/networking/__init__.py b/lib/tfw/networking/__init__.py index c5879b7..c1f3eb4 100644 --- a/lib/tfw/networking/__init__.py +++ b/lib/tfw/networking/__init__.py @@ -1,9 +1,9 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from .serialization import serialize_tfw_msg, deserialize_tfw_msg, with_deserialize_tfw_msg +from .serialization import serialize_tfw_msg, deserialize_tfw_msg +from .serialization import with_deserialize_tfw_msg, message_bytes from .zmq_connector_base import ZMQConnectorBase -# from .controller_connector import ControllerConnector # TODO: readd once controller stuff is resolved from .message_sender import MessageSender from .event_handlers.server_connector import ServerUplinkConnector as TFWServerConnector from .server.tfw_server import TFWServer diff --git a/lib/tfw/networking/serialization.py b/lib/tfw/networking/serialization.py index c28eec0..c54609b 100644 --- a/lib/tfw/networking/serialization.py +++ b/lib/tfw/networking/serialization.py @@ -67,10 +67,14 @@ def _serialize_single(data): (serialize input if it is JSON) """ if not isinstance(data, str): - data = json.dumps(data) + data = message_bytes(data) return _encode_if_needed(data) +def message_bytes(message): + return json.dumps(message, sort_keys=True).encode() + + def _deserialize_single(data): """ Try parsing input as JSON, return it as From eb2c3a8dd081586534069344ed39feefd6f67c58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sun, 15 Jul 2018 17:27:35 +0200 Subject: [PATCH 11/43] Add crypto module with HMAC-SHA256 implementation --- lib/tfw/crypto.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 48 insertions(+) create mode 100644 lib/tfw/crypto.py diff --git a/lib/tfw/crypto.py b/lib/tfw/crypto.py new file mode 100644 index 0000000..413a1d6 --- /dev/null +++ b/lib/tfw/crypto.py @@ -0,0 +1,47 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from functools import wraps + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.hashes import SHA256 +from cryptography.hazmat.primitives.hmac import HMAC as _HMAC +from cryptography.exceptions import InvalidSignature + + +class HMAC: + def __init__(self, key, message): + self.key = key + self.message = message + self._hmac = _HMAC( + key=key, + algorithm=SHA256(), + backend=default_backend() + ) + + def _reload_if_finalized(f): + # pylint: disable=no-self-argument,not-callable + @wraps(f) + def wrapped(instance, *args, **kwargs): + if getattr(instance, '_finalized', False): + instance.__init__(instance.key, instance.message) + ret_val = f(instance, *args, **kwargs) + setattr(instance, '_finalized', True) + return ret_val + return wrapped + + @property + @_reload_if_finalized + def signature(self): + self._hmac.update(self.message) + signature = self._hmac.finalize() + return signature + + @_reload_if_finalized + def verify(self, signature): + self._hmac.update(self.message) + try: + self._hmac.verify(signature) + return True + except InvalidSignature: + return False diff --git a/requirements.txt b/requirements.txt index 4d3a900..2f133de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ terminado==0.8.1 watchdog==0.8.3 PyYAML==3.12 Jinja2==2.10 +cryptography==2.2.2 From b2cb60ef0214b9de8632ecfea752f70ee9f2e3df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sun, 15 Jul 2018 17:29:16 +0200 Subject: [PATCH 12/43] Implement message signing and verification logic --- lib/tfw/crypto.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/tfw/crypto.py b/lib/tfw/crypto.py index 413a1d6..47ab64e 100644 --- a/lib/tfw/crypto.py +++ b/lib/tfw/crypto.py @@ -2,12 +2,32 @@ # All Rights Reserved. See LICENSE file for details. from functools import wraps +from base64 import b64encode, b64decode +from copy import deepcopy from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.hashes import SHA256 from cryptography.hazmat.primitives.hmac import HMAC as _HMAC from cryptography.exceptions import InvalidSignature +from tfw.networking import message_bytes + + +def sign_message(key, message): + signature = HMAC(key, message_bytes(message)).signature + message['signature'] = b64encode(signature).decode() + + +def verify_message(key, message): + message = deepcopy(message) + try: + signature_b64 = message.pop('signature') + signature = b64decode(signature_b64) + actual_signature = HMAC(key, message_bytes(message)).signature + return signature == actual_signature + except KeyError: + return False + class HMAC: def __init__(self, key, message): From 5770e297330e0d6224e8693260934c26953cfcaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sun, 15 Jul 2018 17:30:19 +0200 Subject: [PATCH 13/43] Move message checksum logic to crypto.py --- lib/tfw/crypto.py | 5 +++++ lib/tfw/event_handler_base.py | 14 ++++---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/tfw/crypto.py b/lib/tfw/crypto.py index 47ab64e..be2d54e 100644 --- a/lib/tfw/crypto.py +++ b/lib/tfw/crypto.py @@ -4,6 +4,7 @@ from functools import wraps from base64 import b64encode, b64decode from copy import deepcopy +from hashlib import md5 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.hashes import SHA256 @@ -13,6 +14,10 @@ from cryptography.exceptions import InvalidSignature from tfw.networking import message_bytes +def message_checksum(message): + return md5(message_bytes(message)).hexdigest() + + def sign_message(key, message): signature = HMAC(key, message_bytes(message)).signature message['signature'] = b64encode(signature).decode() diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base.py index f6a094a..0cc023d 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base.py @@ -2,10 +2,9 @@ # All Rights Reserved. See LICENSE file for details. from abc import ABC, abstractmethod -from json import dumps -from hashlib import md5 from tfw.networking.event_handlers import ServerConnector +from tfw.crypto import message_checksum from tfw.config.logs import logging LOG = logging.getLogger(__name__) @@ -82,7 +81,7 @@ class EventHandlerBase(ABC): Note that you can subscribe to the same key several times in which case you will need to unsubscribe multiple times in order to stop receiving events. - + :param keys: list of keys to subscribe to """ for key in keys: @@ -152,7 +151,7 @@ class BroadcastingEventHandler(EventHandlerBase, ABC): self.own_message_hashes = [] def event_handler_callback(self, message): - message_hash = self.hash_message(message) + message_hash = message_checksum(message) if message_hash in self.own_message_hashes: self.own_message_hashes.remove(message_hash) @@ -160,10 +159,5 @@ class BroadcastingEventHandler(EventHandlerBase, ABC): response = self.dispatch_handling(message) if response: - self.own_message_hashes.append(self.hash_message(response)) + self.own_message_hashes.append(message_checksum(response)) self.server_connector.broadcast(response) - - @staticmethod - def hash_message(message): - message_bytes = dumps(message, sort_keys=True).encode() - return md5(message_bytes).hexdigest() From 7c13d31de026f9eab6c5c0d67429a54d2e82d90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 16 Jul 2018 10:29:06 +0200 Subject: [PATCH 14/43] Refactor message signing and verifying logic --- lib/tfw/crypto.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/tfw/crypto.py b/lib/tfw/crypto.py index be2d54e..a40cf5c 100644 --- a/lib/tfw/crypto.py +++ b/lib/tfw/crypto.py @@ -19,16 +19,20 @@ def message_checksum(message): def sign_message(key, message): - signature = HMAC(key, message_bytes(message)).signature + signature = message_signature(key, message) message['signature'] = b64encode(signature).decode() +def message_signature(key, message): + return HMAC(key, message_bytes(message)).signature + + def verify_message(key, message): message = deepcopy(message) try: signature_b64 = message.pop('signature') signature = b64decode(signature_b64) - actual_signature = HMAC(key, message_bytes(message)).signature + actual_signature = message_signature(key, message) return signature == actual_signature except KeyError: return False From a79c68515b9aa6be8a6da3c30f4807a8dfb951c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 16 Jul 2018 11:17:06 +0200 Subject: [PATCH 15/43] Remove controller stuff, tidy code based on pylint suggestions --- .../components/fsm_managing_event_handler.py | 1 + lib/tfw/components/ide_event_handler.py | 4 +- lib/tfw/components/terminal_commands.py | 1 + lib/tfw/event_handler_base.py | 4 +- lib/tfw/fsm_base.py | 1 - lib/tfw/mixins/callback_mixin.py | 1 + lib/tfw/mixins/observer_mixin.py | 1 + lib/tfw/mixins/supervisor_mixin.py | 1 + lib/tfw/networking/controller_connector.py | 18 --------- lib/tfw/networking/server/__init__.py | 1 - .../networking/server/controller_responder.py | 38 ------------------- lib/tfw/networking/server/tfw_server.py | 7 ++-- .../networking/server/zmq_websocket_proxy.py | 2 + lib/tfw/yaml_fsm.py | 28 +++++++------- 14 files changed, 27 insertions(+), 81 deletions(-) delete mode 100644 lib/tfw/networking/controller_connector.py delete mode 100644 lib/tfw/networking/server/controller_responder.py diff --git a/lib/tfw/components/fsm_managing_event_handler.py b/lib/tfw/components/fsm_managing_event_handler.py index 1068bbd..0401a16 100644 --- a/lib/tfw/components/fsm_managing_event_handler.py +++ b/lib/tfw/components/fsm_managing_event_handler.py @@ -33,6 +33,7 @@ class FSMManagingEventHandler(EventHandlerBase): return data def handle_update(self, data): + # pylint: disable=no-self-use return data diff --git a/lib/tfw/components/ide_event_handler.py b/lib/tfw/components/ide_event_handler.py index 72e83b3..2c698ad 100644 --- a/lib/tfw/components/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler.py @@ -4,7 +4,7 @@ from os.path import isfile, join, relpath, exists, isdir, realpath from glob import glob from fnmatch import fnmatchcase -from collections import Iterable +from typing import Iterable from tfw import EventHandlerBase from tfw.mixins import MonitorManagerMixin @@ -103,7 +103,7 @@ class FileManager: # pylint: disable=too-many-instance-attributes class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,anomalous-backslash-in-string """ Event handler implementing the backend of our browser based IDE. By default all files in the directory specified in __init__ are displayed diff --git a/lib/tfw/components/terminal_commands.py b/lib/tfw/components/terminal_commands.py index a47852d..14563bd 100644 --- a/lib/tfw/components/terminal_commands.py +++ b/lib/tfw/components/terminal_commands.py @@ -11,6 +11,7 @@ LOG = logging.getLogger(__name__) class TerminalCommands(ABC): + # pylint: disable=anomalous-backslash-in-string """ A class you can use to define hooks for terminal commands. This means that you can have python code executed when the user enters a specific command to the terminal on diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base.py index 0cc023d..880e406 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base.py @@ -120,8 +120,8 @@ class FSMAwareEventHandler(EventHandlerBase, ABC): def dispatch_handling(self, message): if message['key'] == 'fsm_update': self._handle_fsm_update(message) - else: - return super().dispatch_handling(message) + return None + return super().dispatch_handling(message) def _handle_fsm_update(self, message): try: diff --git a/lib/tfw/fsm_base.py b/lib/tfw/fsm_base.py index 2de2b43..fe3a01a 100644 --- a/lib/tfw/fsm_base.py +++ b/lib/tfw/fsm_base.py @@ -58,7 +58,6 @@ class FSMBase(Machine, CallbackMixin): for predicate in self.trigger_predicates[trigger] ) - # TODO: think about what could we do when this prevents triggering if all(predicate_results): try: self.trigger(trigger) diff --git a/lib/tfw/mixins/callback_mixin.py b/lib/tfw/mixins/callback_mixin.py index 54515f3..33ddb6d 100644 --- a/lib/tfw/mixins/callback_mixin.py +++ b/lib/tfw/mixins/callback_mixin.py @@ -9,6 +9,7 @@ from tfw.decorators import lazy_property class CallbackMixin: @lazy_property def _callbacks(self): + # pylint: disable=no-self-use return [] def subscribe_callback(self, callback, *args, **kwargs): diff --git a/lib/tfw/mixins/observer_mixin.py b/lib/tfw/mixins/observer_mixin.py index 712a31e..9d8b5e2 100644 --- a/lib/tfw/mixins/observer_mixin.py +++ b/lib/tfw/mixins/observer_mixin.py @@ -9,6 +9,7 @@ from tfw.decorators import lazy_property class ObserverMixin: @lazy_property def observer(self): + # pylint: disable=no-self-use return Observer() def watch(self): diff --git a/lib/tfw/mixins/supervisor_mixin.py b/lib/tfw/mixins/supervisor_mixin.py index 00cf398..2238985 100644 --- a/lib/tfw/mixins/supervisor_mixin.py +++ b/lib/tfw/mixins/supervisor_mixin.py @@ -13,6 +13,7 @@ from tfw.config import TFWENV class SupervisorBaseMixin: @lazy_property def supervisor(self): + # pylint: disable=no-self-use return xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor diff --git a/lib/tfw/networking/controller_connector.py b/lib/tfw/networking/controller_connector.py deleted file mode 100644 index b344ce1..0000000 --- a/lib/tfw/networking/controller_connector.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -import zmq -from zmq.eventloop.zmqstream import ZMQStream - -from tfw.config import TFWENV -from tfw.networking import ZMQConnectorBase - - -class ControllerConnector(ZMQConnectorBase): - def __init__(self, zmq_context=None): - super(ControllerConnector, self).__init__(zmq_context) - self._zmq_rep_socket = self._zmq_context.socket(zmq.REP) - self._zmq_rep_socket.connect(f'tcp://localhost:{TFWENV.CONTROLLER_PORT}') - self._zmq_rep_stream = ZMQStream(self._zmq_rep_socket) - - self.register_callback = self._zmq_rep_stream.on_recv_stream diff --git a/lib/tfw/networking/server/__init__.py b/lib/tfw/networking/server/__init__.py index e707fab..eb9adde 100644 --- a/lib/tfw/networking/server/__init__.py +++ b/lib/tfw/networking/server/__init__.py @@ -3,4 +3,3 @@ from .event_handler_connector import EventHandlerConnector, EventHandlerUplinkConnector, EventHandlerDownlinkConnector from .tfw_server import TFWServer -# from .controller_responder import ControllerResponder # TODO: readd once controller stuff is resolved diff --git a/lib/tfw/networking/server/controller_responder.py b/lib/tfw/networking/server/controller_responder.py deleted file mode 100644 index 02d442f..0000000 --- a/lib/tfw/networking/server/controller_responder.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (C) 2018 Avatao.com Innovative Learning Kft. -# All Rights Reserved. See LICENSE file for details. - -from tfw.networking import deserialize_all, serialize_all, ControllerConnector - - -class ControllerResponder: - def __init__(self, fsm): - self.fsm = fsm - self.token = None - self.controller_connector = ControllerConnector() - self.controller_connector.register_callback(self.handle_controller_request) - self.controller_request_handlers = { - 'solution_check': self.handle_solution_check_request, - 'test': self.handle_test_request, - 'token': self.handle_token_request - } - - def handle_controller_request(self, stream, msg_parts): - key, data = deserialize_all(*msg_parts) - response = self.controller_request_handlers[key](data) - stream.send_multipart(serialize_all(self.token, response)) - - def handle_test_request(self, data): - # pylint: disable=unused-argument,no-self-use - return 'OK' - - def handle_token_request(self, data): - if self.token is None: - self.token = data - return {'token': self.token} - - def handle_solution_check_request(self, data): - # pylint: disable=unused-argument - return { - 'solved': self.fsm.is_solved(), - 'message': 'solved' if self.fsm.is_solved() else 'not solved' - } diff --git a/lib/tfw/networking/server/tfw_server.py b/lib/tfw/networking/server/tfw_server.py index 067bb6c..54edbf5 100644 --- a/lib/tfw/networking/server/tfw_server.py +++ b/lib/tfw/networking/server/tfw_server.py @@ -25,12 +25,11 @@ class TFWServer: self._uplink_connector = ServerUplinkConnector() self.application = Application([( - r'/ws', ZMQWebSocketProxy,{ + r'/ws', ZMQWebSocketProxy, { 'event_handler_connector': self._event_handler_connector, 'message_handlers': [self.handle_trigger, self.handle_recover], 'frontend_message_handlers': [self.save_frontend_messages] - })] - ) + })]) self._frontend_messages = FrontendMessageStorage() @@ -68,7 +67,7 @@ class MessageStorage(ABC): def filter_message(self, message): raise NotImplementedError - def transform_message(self, message): + def transform_message(self, message): # pylint: disable=no-self-use yield message diff --git a/lib/tfw/networking/server/zmq_websocket_proxy.py b/lib/tfw/networking/server/zmq_websocket_proxy.py index 7a38e4a..f456d94 100644 --- a/lib/tfw/networking/server/zmq_websocket_proxy.py +++ b/lib/tfw/networking/server/zmq_websocket_proxy.py @@ -12,6 +12,7 @@ LOG = logging.getLogger(__name__) class ZMQWebSocketProxy(WebSocketHandler): + # pylint: disable=abstract-method instances = set() def initialize(self, **kwargs): # pylint: disable=arguments-differ @@ -83,6 +84,7 @@ class ZMQWebSocketProxy(WebSocketHandler): class TFWProxy: + # pylint: disable=protected-access def __init__(self, to_source, to_destination): self.to_source = to_source self.to_destination = to_destination diff --git a/lib/tfw/yaml_fsm.py b/lib/tfw/yaml_fsm.py index 82c955e..63070d2 100644 --- a/lib/tfw/yaml_fsm.py +++ b/lib/tfw/yaml_fsm.py @@ -40,19 +40,19 @@ class YamlFSM(FSMBase): def subscribe_and_remove_predicates(self, json_obj): if 'predicates' in json_obj: for predicate in json_obj['predicates']: - self.subscribe_predicate( - json_obj['trigger'], - partial( - command_statuscode_is_zero, - predicate + self.subscribe_predicate( + json_obj['trigger'], + partial( + command_statuscode_is_zero, + predicate ) - ) - + ) + with suppress(KeyError): json_obj.pop('predicates') -def run_command_async(command, event): +def run_command_async(command, _): Popen(command, shell=True) @@ -62,7 +62,7 @@ def command_statuscode_is_zero(command): class ConfigParser: def __init__(self, config_file, jinja2_variables): - self.read_variables = singledispatch(self.read_variables) + self.read_variables = singledispatch(self._read_variables) self.read_variables.register(dict, self._read_variables_dict) self.read_variables.register(str, self._read_variables_str) @@ -82,16 +82,14 @@ class ConfigParser: return ifile.read() @staticmethod - def read_variables(variables): + def _read_variables(variables): raise TypeError(f'Invalid variables type {type(variables)}') @staticmethod def _read_variables_str(variables): - if isinstance(variables, str): - with open(variables, 'r') as ifile: - return yaml.safe_load(ifile) + with open(variables, 'r') as ifile: + return yaml.safe_load(ifile) @staticmethod def _read_variables_dict(variables): - return variables - + return variables From c28a66fc48883b33ffea50831d1b3dc41763829e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 16 Jul 2018 13:30:53 +0200 Subject: [PATCH 16/43] Make forward slash usage more consistent --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 85f00ee..707a5eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ ENV PYTHONPATH="/usr/local/lib" \ TFW_NGINX_CONF="/etc/nginx/nginx.conf" \ TFW_NGINX_DEFAULT="/etc/nginx/sites-enabled/default" \ TFW_NGINX_COMPONENTS="/etc/nginx/components" \ - TFW_LIB_DIR="/usr/local/lib/" \ + TFW_LIB_DIR="/usr/local/lib" \ TFW_TERMINADO_DIR="/tmp/terminado_server" \ TFW_FRONTEND_DIR="/srv/frontend" \ TFW_SERVER_DIR="/srv/.tfw" \ @@ -50,7 +50,7 @@ COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS} COPY nginx/nginx.conf ${TFW_NGINX_CONF} COPY nginx/default.conf ${TFW_NGINX_DEFAULT} COPY nginx/components/ ${TFW_NGINX_COMPONENTS} -COPY lib LICENSE ${TFW_LIB_DIR} +COPY lib LICENSE ${TFW_LIB_DIR}/ COPY supervisor/tfw_server.py ${TFW_SERVER_DIR}/ RUN for dir in "${TFW_LIB_DIR}"/{tfw,tao,envvars} "/etc/nginx" "/etc/supervisor"; do \ From c658894c12461ac2c726c225171f973b1830de0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 16 Jul 2018 14:28:40 +0200 Subject: [PATCH 17/43] Implement authentication key management --- Dockerfile | 1 + lib/tfw/crypto.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/Dockerfile b/Dockerfile index 707a5eb..8de0107 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,7 @@ ENV PYTHONPATH="/usr/local/lib" \ TFW_TERMINADO_DIR="/tmp/terminado_server" \ TFW_FRONTEND_DIR="/srv/frontend" \ TFW_SERVER_DIR="/srv/.tfw" \ + TFW_AUTH_KEY="/tmp/tfw-auth.key" \ TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history" \ PROMPT_COMMAND="history -a" diff --git a/lib/tfw/crypto.py b/lib/tfw/crypto.py index a40cf5c..af632d8 100644 --- a/lib/tfw/crypto.py +++ b/lib/tfw/crypto.py @@ -5,6 +5,9 @@ from functools import wraps from base64 import b64encode, b64decode from copy import deepcopy from hashlib import md5 +from os import urandom, chmod +from os.path import exists +from stat import S_IRUSR, S_IWUSR, S_IXUSR from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.hashes import SHA256 @@ -12,6 +15,8 @@ from cryptography.hazmat.primitives.hmac import HMAC as _HMAC from cryptography.exceptions import InvalidSignature from tfw.networking import message_bytes +from tfw.decorators import lazy_property +from tfw.config import TFWENV def message_checksum(message): @@ -38,6 +43,32 @@ def verify_message(key, message): return False +class KeyManager: + def __init__(self): + self.keyfile = TFWENV.AUTH_KEY + if not exists(self.keyfile): + self._init_auth_key() + + @lazy_property + def auth_key(self): + with open(self.keyfile, 'rb') as ifile: + return ifile.read() + + def _init_auth_key(self): + key = self.generate_key() + with open(self.keyfile, 'wb') as ofile: + ofile.write(key) + self._chmod_700_keyfile() + return key + + @staticmethod + def generate_key(): + return urandom(32) + + def _chmod_700_keyfile(self): + chmod(self.keyfile, S_IRUSR | S_IWUSR | S_IXUSR) + + class HMAC: def __init__(self, key, message): self.key = key From d5feba707684585dd161d2f6ac3abb605d3a2869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 16 Jul 2018 14:31:52 +0200 Subject: [PATCH 18/43] Implement HMAC signatures of fsm_update broadcast messages --- lib/tfw/components/fsm_managing_event_handler.py | 7 ++++++- lib/tfw/event_handler_base.py | 6 ++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/tfw/components/fsm_managing_event_handler.py b/lib/tfw/components/fsm_managing_event_handler.py index 0401a16..4e2c97f 100644 --- a/lib/tfw/components/fsm_managing_event_handler.py +++ b/lib/tfw/components/fsm_managing_event_handler.py @@ -2,6 +2,7 @@ # All Rights Reserved. See LICENSE file for details. from tfw import EventHandlerBase +from tfw.crypto import KeyManager, sign_message from tfw.config.logs import logging LOG = logging.getLogger(__name__) @@ -12,6 +13,7 @@ class FSMManagingEventHandler(EventHandlerBase): super().__init__(key) self.fsm = fsm_type() self._fsm_updater = FSMUpdater(self.fsm) + self.auth_key = KeyManager().auth_key self.command_handlers = { 'trigger': self.handle_trigger, @@ -22,7 +24,10 @@ class FSMManagingEventHandler(EventHandlerBase): try: data = message['data'] message['data'] = self.command_handlers[data['command']](data) - self.server_connector.broadcast(self._fsm_updater.generate_fsm_update()) + fsm_update_message = self._fsm_updater.generate_fsm_update() + sign_message(self.auth_key, message) + sign_message(self.auth_key, fsm_update_message) + self.server_connector.broadcast(fsm_update_message) return message except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base.py index 880e406..fb99748 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from tfw.networking.event_handlers import ServerConnector -from tfw.crypto import message_checksum +from tfw.crypto import message_checksum, KeyManager, verify_message from tfw.config.logs import logging LOG = logging.getLogger(__name__) @@ -116,10 +116,12 @@ class FSMAwareEventHandler(EventHandlerBase, ABC): super().__init__(key) self.subscribe('fsm_update') self.fsm_state = None + self._auth_key = KeyManager().auth_key def dispatch_handling(self, message): if message['key'] == 'fsm_update': - self._handle_fsm_update(message) + if verify_message(self._auth_key, message): + self._handle_fsm_update(message) return None return super().dispatch_handling(message) From 8e8702590518fdebaa9e1fe601f0eb7176150e89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 17 Jul 2018 16:04:28 +0200 Subject: [PATCH 19/43] Implement git based fs snapshot provider --- lib/tfw/components/__init__.py | 1 + lib/tfw/components/snapshot_provider.py | 54 +++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 lib/tfw/components/snapshot_provider.py diff --git a/lib/tfw/components/__init__.py b/lib/tfw/components/__init__.py index ad160f1..4c0475c 100644 --- a/lib/tfw/components/__init__.py +++ b/lib/tfw/components/__init__.py @@ -9,3 +9,4 @@ from .history_monitor import HistoryMonitor, BashMonitor, GDBMonitor from .terminal_commands import TerminalCommands from .log_monitoring_event_handler import LogMonitoringEventHandler from .fsm_managing_event_handler import FSMManagingEventHandler +from .snapshot_provider import SnapshotProvider diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py new file mode 100644 index 0000000..e240b02 --- /dev/null +++ b/lib/tfw/components/snapshot_provider.py @@ -0,0 +1,54 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from subprocess import run +from getpass import getuser + + +class SnapshotProvider: + def __init__(self, directory, git_dir): + author = f'{getuser()} via TFW {self.__class__.__name__}' + self.gitenv = { + 'GIT_DIR': git_dir, + 'GIT_WORK_TREE': directory, + 'GIT_AUTHOR_NAME': author, + 'GIT_AUTHOR_EMAIL': '', + 'GIT_COMMITTER_NAME': author, + 'GIT_COMMITTER_EMAIL': '' + } + + def init_repo(self): + self._run(('git', 'init')) + + def take_snapshot(self): + self._run(('git', 'add', '-A')) + self._run(('git', 'commit', '-m', 'Snapshot')) + + def restore_snapshot(self, date): + commit = self._get_commit_from_timestamp(date) + self._checkout_commit(commit) + + def _get_commit_from_timestamp(self, date): + return self._get_stdout(( + 'git', 'rev-list', + '--date=iso', + '-n', '1', + f'--before="{date.isoformat()}"', + 'master' + )) + + def _checkout_commit(self, commit): + self._run(( + 'git', 'checkout', + commit + )) + + def _get_stdout(self, *args, **kwargs): + kwargs['capture_output'] = True + stdout_bytes = self._run(*args, **kwargs).stdout + return stdout_bytes.decode().rstrip('\n') + + def _run(self, *args, **kwargs): + if 'env' not in kwargs: + kwargs['env'] = self.gitenv + return run(*args, **kwargs) From c279b2517fd045fcc5e9aa06ce65883eec2df49b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 18 Jul 2018 13:38:17 +0200 Subject: [PATCH 20/43] Implement branching in SnapshotProvider --- lib/tfw/components/snapshot_provider.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index e240b02..1e9f226 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -17,16 +17,36 @@ class SnapshotProvider: 'GIT_COMMITTER_EMAIL': '' } + self._head_detached = False + self._branch = 'master' + self._branches = [self._branch] + def init_repo(self): self._run(('git', 'init')) def take_snapshot(self): + if self._head_detached: + self._checkout_branch_from_head() self._run(('git', 'add', '-A')) self._run(('git', 'commit', '-m', 'Snapshot')) + def _checkout_branch_from_head(self): + head_hash = self._get_head_hash() + self._run(( + 'git', 'checkout', + '-b', head_hash, head_hash + )) + self._branches.append(head_hash) + self._branch = head_hash + self._head_detached = False + + def _get_head_hash(self): + return self._get_stdout(('git', 'rev-parse', 'HEAD')) + def restore_snapshot(self, date): commit = self._get_commit_from_timestamp(date) self._checkout_commit(commit) + self._head_detached = True def _get_commit_from_timestamp(self, date): return self._get_stdout(( @@ -34,7 +54,7 @@ class SnapshotProvider: '--date=iso', '-n', '1', f'--before="{date.isoformat()}"', - 'master' + self._branch )) def _checkout_commit(self, commit): From 2f5e8d11f353a643e84bf83fd6ac0757606b81d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 18 Jul 2018 13:42:05 +0200 Subject: [PATCH 21/43] Automatically init git in SnapshotProvider --- lib/tfw/components/snapshot_provider.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 1e9f226..65517e8 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -21,7 +21,9 @@ class SnapshotProvider: self._branch = 'master' self._branches = [self._branch] - def init_repo(self): + self._init_repo() + + def _init_repo(self): self._run(('git', 'init')) def take_snapshot(self): From 49e0efa4c903e99e51cc0aead0e95129165d7fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 18 Jul 2018 14:15:19 +0200 Subject: [PATCH 22/43] Run git init only if there is no repo present --- lib/tfw/components/snapshot_provider.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 65517e8..0ce8133 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -21,10 +21,14 @@ class SnapshotProvider: self._branch = 'master' self._branches = [self._branch] - self._init_repo() + self._init_repo_if_needed() - def _init_repo(self): - self._run(('git', 'init')) + def _init_repo_if_needed(self): + if not self._repo_is_initialized(): + self._run(('git', 'init')) + + def _repo_is_initialized(self): + return self._run(('git', 'status')).returncode == 0 def take_snapshot(self): if self._head_detached: From e80782aa6c547c00e489123a613947ec2627d15b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 18 Jul 2018 14:26:14 +0200 Subject: [PATCH 23/43] Implement checking whether directories passed exsist --- lib/tfw/components/snapshot_provider.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 0ce8133..53cb63e 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -3,6 +3,7 @@ from subprocess import run from getpass import getuser +from os.path import isdir class SnapshotProvider: @@ -17,12 +18,18 @@ class SnapshotProvider: 'GIT_COMMITTER_EMAIL': '' } + self._check_environment() + self._head_detached = False self._branch = 'master' self._branches = [self._branch] self._init_repo_if_needed() + def _check_environment(self): + if not isdir(self.gitenv['GIT_DIR']) or not isdir(self.gitenv['GIT_WORK_TREE']): + raise EnvironmentError('Directories "directory" and "git_dir" must exist!') + def _init_repo_if_needed(self): if not self._repo_is_initialized(): self._run(('git', 'init')) From d306d0e484bcb2cf235e2458f57710d2681d801a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 18 Jul 2018 15:47:12 +0200 Subject: [PATCH 24/43] Implement properties to get info from SnapshotProvider instance --- lib/tfw/components/snapshot_provider.py | 42 ++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 53cb63e..1a11c51 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -4,6 +4,7 @@ from subprocess import run from getpass import getuser from os.path import isdir +from datetime import datetime class SnapshotProvider: @@ -15,7 +16,8 @@ class SnapshotProvider: 'GIT_AUTHOR_NAME': author, 'GIT_AUTHOR_EMAIL': '', 'GIT_COMMITTER_NAME': author, - 'GIT_COMMITTER_EMAIL': '' + 'GIT_COMMITTER_EMAIL': '', + 'GIT_PAGER': 'cat' } self._check_environment() @@ -58,7 +60,7 @@ class SnapshotProvider: def restore_snapshot(self, date): commit = self._get_commit_from_timestamp(date) - self._checkout_commit(commit) + self._checkout(commit) self._head_detached = True def _get_commit_from_timestamp(self, date): @@ -70,10 +72,10 @@ class SnapshotProvider: self._branch )) - def _checkout_commit(self, commit): + def _checkout(self, what): self._run(( 'git', 'checkout', - commit + what )) def _get_stdout(self, *args, **kwargs): @@ -85,3 +87,35 @@ class SnapshotProvider: if 'env' not in kwargs: kwargs['env'] = self.gitenv return run(*args, **kwargs) + + @property + def all_timelines(self): + return self._branches + + @property + def timeline(self): + return self._branch + + @timeline.setter + def timeline(self, value): + self._checkout(value) + + @property + def snapshots(self): + return self._pretty_log_branch() + + def _pretty_log_branch(self): + git_log_output = self._get_stdout(( + 'git', 'log', + '--pretty=%H@%aI' + )) + + commits = [] + for line in git_log_output.splitlines(): + commit_hash, timestamp = line.split('@') + commits.append({ + 'hash': commit_hash, + 'timestamp': datetime.fromisoformat(timestamp) + }) + + return commits From fb2beb470f920b38d9242540cabf5fac9a9942bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 18 Jul 2018 15:50:35 +0200 Subject: [PATCH 25/43] Make subprocess calls fail if something goes wrong --- lib/tfw/components/snapshot_provider.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 1a11c51..96ea772 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -84,6 +84,7 @@ class SnapshotProvider: return stdout_bytes.decode().rstrip('\n') def _run(self, *args, **kwargs): + kwargs['check'] = True if 'env' not in kwargs: kwargs['env'] = self.gitenv return run(*args, **kwargs) From fa3ce317f063bf49a775db75e0a3d28a96ff9814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 18 Jul 2018 16:26:10 +0200 Subject: [PATCH 26/43] Rename stuff to improve code readability --- lib/tfw/components/snapshot_provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 96ea772..184e96c 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -41,11 +41,11 @@ class SnapshotProvider: def take_snapshot(self): if self._head_detached: - self._checkout_branch_from_head() + self._checkout_new_branch_from_head() self._run(('git', 'add', '-A')) self._run(('git', 'commit', '-m', 'Snapshot')) - def _checkout_branch_from_head(self): + def _checkout_new_branch_from_head(self): head_hash = self._get_head_hash() self._run(( 'git', 'checkout', From 85523ede32576b861523ebf187cfb56d9a35b2bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 19 Jul 2018 11:06:13 +0200 Subject: [PATCH 27/43] Make SnapshotProvider more in line with reality --- lib/tfw/components/snapshot_provider.py | 39 ++++++++++++++++++------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 184e96c..052966b 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -1,6 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +import re from subprocess import run from getpass import getuser from os.path import isdir @@ -21,16 +22,14 @@ class SnapshotProvider: } self._check_environment() - - self._head_detached = False - self._branch = 'master' - self._branches = [self._branch] - + self.__last_valid_branch = self._branch self._init_repo_if_needed() def _check_environment(self): if not isdir(self.gitenv['GIT_DIR']) or not isdir(self.gitenv['GIT_WORK_TREE']): raise EnvironmentError('Directories "directory" and "git_dir" must exist!') + if self._head_detached: + raise EnvironmentError(f'{self.__class__.__name__} cannot init from detached HEAD state!') def _init_repo_if_needed(self): if not self._repo_is_initialized(): @@ -51,9 +50,6 @@ class SnapshotProvider: 'git', 'checkout', '-b', head_hash, head_hash )) - self._branches.append(head_hash) - self._branch = head_hash - self._head_detached = False def _get_head_hash(self): return self._get_stdout(('git', 'rev-parse', 'HEAD')) @@ -61,7 +57,6 @@ class SnapshotProvider: def restore_snapshot(self, date): commit = self._get_commit_from_timestamp(date) self._checkout(commit) - self._head_detached = True def _get_commit_from_timestamp(self, date): return self._get_stdout(( @@ -69,7 +64,7 @@ class SnapshotProvider: '--date=iso', '-n', '1', f'--before="{date.isoformat()}"', - self._branch + self._last_valid_branch )) def _checkout(self, what): @@ -93,10 +88,34 @@ class SnapshotProvider: def all_timelines(self): return self._branches + @property + def _branches(self): + git_branch_output = self._get_stdout(('git', 'branch')) + regex_pattern = re.compile(r'(?:[^\S\n]|[*])') # matches '*' and non-newline whitespace chars + return re.sub(regex_pattern, '', git_branch_output).splitlines() + @property def timeline(self): return self._branch + @property + def _branch(self): + return self._get_stdout(( + 'git', 'rev-parse', + '--abbrev-ref', 'HEAD' + )) + + @property + def _last_valid_branch(self): + current_branch = self._branch + if current_branch != 'HEAD': + self.__last_valid_branch = current_branch + return self.__last_valid_branch + + @property + def _head_detached(self): + return self._branch == 'HEAD' + @timeline.setter def timeline(self, value): self._checkout(value) From dd056a37bb62b59c3f304fe78478cf57a7cc5efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 19 Jul 2018 11:11:56 +0200 Subject: [PATCH 28/43] Make coding style more consistent --- lib/tfw/components/snapshot_provider.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 052966b..e051238 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -41,18 +41,28 @@ class SnapshotProvider: def take_snapshot(self): if self._head_detached: self._checkout_new_branch_from_head() - self._run(('git', 'add', '-A')) - self._run(('git', 'commit', '-m', 'Snapshot')) + self._run(( + 'git', 'add', + '-A' + )) + self._run(( + 'git', 'commit', + '-m', 'Snapshot' + )) def _checkout_new_branch_from_head(self): head_hash = self._get_head_hash() self._run(( 'git', 'checkout', - '-b', head_hash, head_hash + '-b', head_hash, + head_hash )) def _get_head_hash(self): - return self._get_stdout(('git', 'rev-parse', 'HEAD')) + return self._get_stdout(( + 'git', 'rev-parse', + 'HEAD' + )) def restore_snapshot(self, date): commit = self._get_commit_from_timestamp(date) From c6d3b8ad214db83868e55f8847d009f278bc7248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 19 Jul 2018 11:31:31 +0200 Subject: [PATCH 29/43] Ensure _last_valid_branch consistency --- lib/tfw/components/snapshot_provider.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index e051238..e0a90ad 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -53,10 +53,10 @@ class SnapshotProvider: def _checkout_new_branch_from_head(self): head_hash = self._get_head_hash() self._run(( - 'git', 'checkout', - '-b', head_hash, + 'git', 'branch', head_hash )) + self._checkout(head_hash) def _get_head_hash(self): return self._get_stdout(( @@ -68,6 +68,13 @@ class SnapshotProvider: commit = self._get_commit_from_timestamp(date) self._checkout(commit) + def _checkout(self, what): + self._run(( + 'git', 'checkout', + what + )) + self._update_last_valid_branch() + def _get_commit_from_timestamp(self, date): return self._get_stdout(( 'git', 'rev-list', @@ -77,12 +84,6 @@ class SnapshotProvider: self._last_valid_branch )) - def _checkout(self, what): - self._run(( - 'git', 'checkout', - what - )) - def _get_stdout(self, *args, **kwargs): kwargs['capture_output'] = True stdout_bytes = self._run(*args, **kwargs).stdout @@ -117,11 +118,13 @@ class SnapshotProvider: @property def _last_valid_branch(self): - current_branch = self._branch - if current_branch != 'HEAD': - self.__last_valid_branch = current_branch + self._update_last_valid_branch() return self.__last_valid_branch + def _update_last_valid_branch(self): + if not self._head_detached: + self.__last_valid_branch = self._branch + @property def _head_detached(self): return self._branch == 'HEAD' From d8ac0dc311aba274c76011dab605bcb87c0cc2eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 19 Jul 2018 11:37:51 +0200 Subject: [PATCH 30/43] Reorder SnapshotProvider methods in accordance with Uncle Bob --- lib/tfw/components/snapshot_provider.py | 72 ++++++++++++------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index e0a90ad..cfb547e 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -31,6 +31,28 @@ class SnapshotProvider: if self._head_detached: raise EnvironmentError(f'{self.__class__.__name__} cannot init from detached HEAD state!') + @property + def _head_detached(self): + return self._branch == 'HEAD' + + @property + def _branch(self): + return self._get_stdout(( + 'git', 'rev-parse', + '--abbrev-ref', 'HEAD' + )) + + def _get_stdout(self, *args, **kwargs): + kwargs['capture_output'] = True + stdout_bytes = self._run(*args, **kwargs).stdout + return stdout_bytes.decode().rstrip('\n') + + def _run(self, *args, **kwargs): + kwargs['check'] = True + if 'env' not in kwargs: + kwargs['env'] = self.gitenv + return run(*args, **kwargs) + def _init_repo_if_needed(self): if not self._repo_is_initialized(): self._run(('git', 'init')) @@ -68,13 +90,6 @@ class SnapshotProvider: commit = self._get_commit_from_timestamp(date) self._checkout(commit) - def _checkout(self, what): - self._run(( - 'git', 'checkout', - what - )) - self._update_last_valid_branch() - def _get_commit_from_timestamp(self, date): return self._get_stdout(( 'git', 'rev-list', @@ -84,16 +99,21 @@ class SnapshotProvider: self._last_valid_branch )) - def _get_stdout(self, *args, **kwargs): - kwargs['capture_output'] = True - stdout_bytes = self._run(*args, **kwargs).stdout - return stdout_bytes.decode().rstrip('\n') + @property + def _last_valid_branch(self): + self._update_last_valid_branch() + return self.__last_valid_branch - def _run(self, *args, **kwargs): - kwargs['check'] = True - if 'env' not in kwargs: - kwargs['env'] = self.gitenv - return run(*args, **kwargs) + def _update_last_valid_branch(self): + if not self._head_detached: + self.__last_valid_branch = self._branch + + def _checkout(self, what): + self._run(( + 'git', 'checkout', + what + )) + self._update_last_valid_branch() @property def all_timelines(self): @@ -109,26 +129,6 @@ class SnapshotProvider: def timeline(self): return self._branch - @property - def _branch(self): - return self._get_stdout(( - 'git', 'rev-parse', - '--abbrev-ref', 'HEAD' - )) - - @property - def _last_valid_branch(self): - self._update_last_valid_branch() - return self.__last_valid_branch - - def _update_last_valid_branch(self): - if not self._head_detached: - self.__last_valid_branch = self._branch - - @property - def _head_detached(self): - return self._branch == 'HEAD' - @timeline.setter def timeline(self, value): self._checkout(value) From 98e1bbe9d6b7bf5127158b7a264165c791625353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 19 Jul 2018 14:25:25 +0200 Subject: [PATCH 31/43] Remove unnecessary _last_valid_tag stuff --- lib/tfw/components/snapshot_provider.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index cfb547e..01c1624 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -101,19 +101,15 @@ class SnapshotProvider: @property def _last_valid_branch(self): - self._update_last_valid_branch() - return self.__last_valid_branch - - def _update_last_valid_branch(self): if not self._head_detached: self.__last_valid_branch = self._branch + return self.__last_valid_branch def _checkout(self, what): self._run(( 'git', 'checkout', what )) - self._update_last_valid_branch() @property def all_timelines(self): From 889ec92928e74f778275186bc4f8892a6e2d40b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 19 Jul 2018 14:26:41 +0200 Subject: [PATCH 32/43] Use UUIDs instead of commit hashes for new branch names (git ambigous) --- lib/tfw/components/snapshot_provider.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 01c1624..ecc4d2a 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -6,6 +6,7 @@ from subprocess import run from getpass import getuser from os.path import isdir from datetime import datetime +from uuid import uuid4 class SnapshotProvider: @@ -73,18 +74,12 @@ class SnapshotProvider: )) def _checkout_new_branch_from_head(self): - head_hash = self._get_head_hash() + branch_name = uuid4() self._run(( 'git', 'branch', - head_hash - )) - self._checkout(head_hash) - - def _get_head_hash(self): - return self._get_stdout(( - 'git', 'rev-parse', - 'HEAD' + branch_name )) + self._checkout(branch_name) def restore_snapshot(self, date): commit = self._get_commit_from_timestamp(date) From 1bb6286d241f38b2289c021e68fccecf748fcec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 19 Jul 2018 14:29:14 +0200 Subject: [PATCH 33/43] Fix timeline property --- lib/tfw/components/snapshot_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index ecc4d2a..4ce61eb 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -118,7 +118,7 @@ class SnapshotProvider: @property def timeline(self): - return self._branch + return self._last_valid_branch @timeline.setter def timeline(self, value): From a703ee821d678b3d692a0b1accae476197872d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 19 Jul 2018 14:56:59 +0200 Subject: [PATCH 34/43] Fix SnapshotProvider initialization stuff --- lib/tfw/components/snapshot_provider.py | 62 ++++++++++++++++++------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 4ce61eb..0b3b008 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -22,13 +22,52 @@ class SnapshotProvider: 'GIT_PAGER': 'cat' } - self._check_environment() + self._init_repo() self.__last_valid_branch = self._branch - self._init_repo_if_needed() + + def _init_repo(self): + self._check_environment() + + if not self._repo_is_initialized: + self._run(('git', 'init')) + + if self._number_of_commits == 0: + self._snapshot() + + self._check_head_not_detached() def _check_environment(self): if not isdir(self.gitenv['GIT_DIR']) or not isdir(self.gitenv['GIT_WORK_TREE']): raise EnvironmentError('Directories "directory" and "git_dir" must exist!') + + @property + def _repo_is_initialized(self): + return self._run( + ('git', 'status'), + check=False + ).returncode == 0 + + @property + def _number_of_commits(self): + return int( + self._get_stdout(( + 'git', 'rev-list', + '--all', + '--count' + )) + ) + + def _snapshot(self): + self._run(( + 'git', 'add', + '-A' + )) + self._run(( + 'git', 'commit', + '-m', 'Snapshot' + )) + + def _check_head_not_detached(self): if self._head_detached: raise EnvironmentError(f'{self.__class__.__name__} cannot init from detached HEAD state!') @@ -49,29 +88,16 @@ class SnapshotProvider: return stdout_bytes.decode().rstrip('\n') def _run(self, *args, **kwargs): - kwargs['check'] = True + if 'check' not in kwargs: + kwargs['check'] = True if 'env' not in kwargs: kwargs['env'] = self.gitenv return run(*args, **kwargs) - def _init_repo_if_needed(self): - if not self._repo_is_initialized(): - self._run(('git', 'init')) - - def _repo_is_initialized(self): - return self._run(('git', 'status')).returncode == 0 - def take_snapshot(self): if self._head_detached: self._checkout_new_branch_from_head() - self._run(( - 'git', 'add', - '-A' - )) - self._run(( - 'git', 'commit', - '-m', 'Snapshot' - )) + self._snapshot() def _checkout_new_branch_from_head(self): branch_name = uuid4() From 0dd66c33bffe5e247e86fe86984c5878abede9d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 19 Jul 2018 15:05:43 +0200 Subject: [PATCH 35/43] Make error handling more robust --- lib/tfw/components/snapshot_provider.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 0b3b008..69aa33e 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -2,7 +2,7 @@ # All Rights Reserved. See LICENSE file for details. import re -from subprocess import run +from subprocess import run, CalledProcessError from getpass import getuser from os.path import isdir from datetime import datetime @@ -11,7 +11,8 @@ from uuid import uuid4 class SnapshotProvider: def __init__(self, directory, git_dir): - author = f'{getuser()} via TFW {self.__class__.__name__}' + self._classname = self.__class__.__name__ + author = f'{getuser()} via TFW {self._classname}' self.gitenv = { 'GIT_DIR': git_dir, 'GIT_WORK_TREE': directory, @@ -32,13 +33,16 @@ class SnapshotProvider: self._run(('git', 'init')) if self._number_of_commits == 0: - self._snapshot() + try: + self._snapshot() + except CalledProcessError: + raise EnvironmentError(f'{self._classname} cannot init on empty directories!') self._check_head_not_detached() def _check_environment(self): if not isdir(self.gitenv['GIT_DIR']) or not isdir(self.gitenv['GIT_WORK_TREE']): - raise EnvironmentError('Directories "directory" and "git_dir" must exist!') + raise EnvironmentError(f'{self._classname}: "directory" and "git_dir" must exist!') @property def _repo_is_initialized(self): @@ -69,7 +73,7 @@ class SnapshotProvider: def _check_head_not_detached(self): if self._head_detached: - raise EnvironmentError(f'{self.__class__.__name__} cannot init from detached HEAD state!') + raise EnvironmentError(f'{self._classname} cannot init from detached HEAD state!') @property def _head_detached(self): From a8fdd3b0d8620f2e0356d4da7803936c993fea8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 19 Jul 2018 16:15:54 +0200 Subject: [PATCH 36/43] Refactor SnapshotProvider --- lib/tfw/components/snapshot_provider.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index 69aa33e..edee8ab 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -111,6 +111,12 @@ class SnapshotProvider: )) self._checkout(branch_name) + def _checkout(self, what): + self._run(( + 'git', 'checkout', + what + )) + def restore_snapshot(self, date): commit = self._get_commit_from_timestamp(date) self._checkout(commit) @@ -130,12 +136,6 @@ class SnapshotProvider: self.__last_valid_branch = self._branch return self.__last_valid_branch - def _checkout(self, what): - self._run(( - 'git', 'checkout', - what - )) - @property def all_timelines(self): return self._branches From 7376504dd2d642ca83d89d413794bc94bdd5c7f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 20 Jul 2018 09:16:06 +0200 Subject: [PATCH 37/43] Make frontend state recovery recover selected file in IDE --- lib/tfw/networking/server/tfw_server.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/tfw/networking/server/tfw_server.py b/lib/tfw/networking/server/tfw_server.py index 54edbf5..41bb7aa 100644 --- a/lib/tfw/networking/server/tfw_server.py +++ b/lib/tfw/networking/server/tfw_server.py @@ -73,7 +73,12 @@ class MessageStorage(ABC): class FrontendMessageStorage(MessageStorage): def filter_message(self, message): - return message['key'] in ('message', 'dashboard', 'queueMessages') + key = message['key'] + command = message.get('data', {}).get('command') + return ( + key in ('message', 'dashboard', 'queueMessages') + or key == 'ide' and command in ('select', 'read') + ) def transform_message(self, message): if message['key'] == 'queueMessages': From bd6cdc5409f4d0cc5edc384527ca995c5c6b76d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 20 Jul 2018 09:20:42 +0200 Subject: [PATCH 38/43] Fix stored messages growing infinitely on reloads --- lib/tfw/networking/server/tfw_server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/tfw/networking/server/tfw_server.py b/lib/tfw/networking/server/tfw_server.py index 41bb7aa..f0471ef 100644 --- a/lib/tfw/networking/server/tfw_server.py +++ b/lib/tfw/networking/server/tfw_server.py @@ -47,6 +47,7 @@ class TFWServer: def handle_recover(self, message): if message['key'] == 'recover': self._frontend_messages.replay_messages(self._uplink_connector) + self._frontend_messages.clear() def save_frontend_messages(self, message): self._frontend_messages.save_message(message) @@ -70,6 +71,9 @@ class MessageStorage(ABC): def transform_message(self, message): # pylint: disable=no-self-use yield message + def clear(self): + self.saved_messages.clear() + class FrontendMessageStorage(MessageStorage): def filter_message(self, message): From 3e5dd6e1021029732fc98676dbb893c704c4358f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 20 Jul 2018 10:27:58 +0200 Subject: [PATCH 39/43] Fix typo --- lib/tfw/networking/message_sender.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tfw/networking/message_sender.py b/lib/tfw/networking/message_sender.py index 2b946d7..378ad91 100644 --- a/lib/tfw/networking/message_sender.py +++ b/lib/tfw/networking/message_sender.py @@ -46,8 +46,8 @@ class MessageSender: }) @staticmethod - def generate_messages_from_queue(queue_messsge): - for message in queue_messsge['data']['messages']: + def generate_messages_from_queue(queue_message): + for message in queue_message['data']['messages']: yield { 'key': 'message', 'data': message From 446d453947f574e776665298c6e9d5ff6d80a8c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 20 Jul 2018 14:37:26 +0200 Subject: [PATCH 40/43] Make TFW lib code location a volume so controller can use it --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8de0107..a264278 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,7 +67,7 @@ ONBUILD COPY ${BUILD_CONTEXT}/supervisor/ ${TFW_SUPERVISORD_COMPONENTS} ONBUILD RUN for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do \ envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f~ && mv $f~ $f ;\ done -ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx"] +ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx", "${TFW_LIB_DIR}/envvars", "${TFW_LIB_DIR}/tfw"] ONBUILD COPY ${BUILD_CONTEXT}/frontend /data/ ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn install --frozen-lockfile || : From e1e0acb3b86fa77f343bebc3d12953e6507da608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 20 Jul 2018 14:38:26 +0200 Subject: [PATCH 41/43] Fix FSMBase default accepted_state --- lib/tfw/fsm_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/fsm_base.py b/lib/tfw/fsm_base.py index fe3a01a..3cc2af5 100644 --- a/lib/tfw/fsm_base.py +++ b/lib/tfw/fsm_base.py @@ -22,7 +22,7 @@ class FSMBase(Machine, CallbackMixin): states, transitions = [], [] def __init__(self, initial=None, accepted_states=None): - self.accepted_states = accepted_states or [self.states[-1]] + self.accepted_states = accepted_states or [self.states[-1].name] self.trigger_predicates = defaultdict(list) self.trigger_history = [] From 96b4e314a920eae7f52037152c86dcdafdf843da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 20 Jul 2018 14:39:23 +0200 Subject: [PATCH 42/43] Add 'in_accepted_state' to fsm_update messages --- lib/tfw/components/fsm_managing_event_handler.py | 4 +++- lib/tfw/event_handler_base.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/tfw/components/fsm_managing_event_handler.py b/lib/tfw/components/fsm_managing_event_handler.py index 4e2c97f..ee95c6f 100644 --- a/lib/tfw/components/fsm_managing_event_handler.py +++ b/lib/tfw/components/fsm_managing_event_handler.py @@ -59,8 +59,10 @@ class FSMUpdater: for trigger in self.fsm.get_triggers(self.fsm.state) ] last_trigger = self.fsm.trigger_history[-1] if self.fsm.trigger_history else None + in_accepted_state = state in self.fsm.accepted_states return { 'current_state': state, 'valid_transitions': valid_transitions, - 'last_trigger': last_trigger + 'last_trigger': last_trigger, + 'in_accepted_state': in_accepted_state } diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base.py index fb99748..aaf408c 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base.py @@ -116,6 +116,7 @@ class FSMAwareEventHandler(EventHandlerBase, ABC): super().__init__(key) self.subscribe('fsm_update') self.fsm_state = None + self.in_accepted_state = False self._auth_key = KeyManager().auth_key def dispatch_handling(self, message): @@ -132,6 +133,7 @@ class FSMAwareEventHandler(EventHandlerBase, ABC): if self.fsm_state != new_state: self.handle_fsm_step(self.fsm_state, new_state, trigger) self.fsm_state = new_state + self.in_accepted_state = message['data']['in_accepted_state'] except KeyError: LOG.error('Invalid fsm_update message received!') From cb4ba563e99f4a91df31f2751f949e05b2210f8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 20 Jul 2018 15:03:28 +0200 Subject: [PATCH 43/43] Fix LinearFSM not being compatible with controller stuff --- lib/tfw/linear_fsm.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/tfw/linear_fsm.py b/lib/tfw/linear_fsm.py index 1f0e3af..a053c30 100644 --- a/lib/tfw/linear_fsm.py +++ b/lib/tfw/linear_fsm.py @@ -1,6 +1,8 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. +from transitions import State + from .fsm_base import FSMBase @@ -13,17 +15,17 @@ class LinearFSM(FSMBase): (0) -- step_1 --> (1) -- step_2 --> (2) -- step_3 --> (3) ... and so on """ def __init__(self, number_of_steps): - self.states = list(map(str, range(number_of_steps))) + self.states = [State(name=str(index)) for index in range(number_of_steps)] self.transitions = [] - for index in self.states[:-1]: + for state in self.states[:-1]: self.transitions.append({ - 'trigger': f'step_{int(index)+1}', - 'source': index, - 'dest': str(int(index)+1) + 'trigger': f'step_{int(state.name)+1}', + 'source': state.name, + 'dest': str(int(state.name)+1) }) self.transitions.append({ 'trigger': 'step_next', - 'source': index, - 'dest': str(int(index)+1) + 'source': state.name, + 'dest': str(int(state.name)+1) }) super(LinearFSM, self).__init__()