diff --git a/.drone.yml b/.drone.yml index e49f310..d998013 100644 --- a/.drone.yml +++ b/.drone.yml @@ -10,4 +10,4 @@ pipeline: - docker push eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG} when: event: 'tag' - branch: refs/tags/mainecoon-20* + branch: refs/tags/ocicat-20* diff --git a/.git-hooks/apply_hooks.sh b/.git-hooks/apply_hooks.sh new file mode 100755 index 0000000..f95f8f4 --- /dev/null +++ b/.git-hooks/apply_hooks.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -eu +set -o pipefail +set -o errtrace +shopt -s expand_aliases + +[ "$(uname)" == "Darwin" ] && alias readlink="greadlink" || : + +GREEN='\033[0;32m' +NC='\033[0m' + + +here="$(dirname "$(readlink -f "$0")")" +cd "${here}/../.git/hooks" + +rm -f pre-push pre-commit || : +prepush_script="../../.git-hooks/pre-push.sh" +precommit_script="../../.git-hooks/pre-commit.sh" +[ -f "${prepush_script}" ] && ln -s "${prepush_script}" pre-push +[ -f "${precommit_script}" ] && ln -s "${precommit_script}" pre-commit + +echo -e "\n${GREEN}Done! Hooks applied, you can start committing and pushing!${NC}\n" diff --git a/.git-hooks/pre-push.sh b/.git-hooks/pre-push.sh new file mode 100755 index 0000000..e2025ac --- /dev/null +++ b/.git-hooks/pre-push.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -eu +set -o pipefail +set -o errtrace +shopt -s expand_aliases + +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' + + +echo -e "Running pylint...\n" +if pylint lib; then + echo -e "\n${GREEN}Pylint found no errors!${NC}\n" +else + echo -e "\n${RED}Pylint failed with errors${NC}\n" + exit 1 +fi diff --git a/.pylintrc b/.pylintrc index f36e237..fd195bc 100644 --- a/.pylintrc +++ b/.pylintrc @@ -3,3 +3,9 @@ ignored-modules = zmq max-line-length = 120 disable = missing-docstring, too-few-public-methods, invalid-name + +[SIMILARITIES] + +ignore-comments=yes +ignore-docstrings=yes +ignore-imports=yes diff --git a/Dockerfile b/Dockerfile index f9ccf5e..22237e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,9 +36,10 @@ ENV PYTHONPATH="/usr/local/lib" \ TFW_NGINX_DEFAULT="/etc/nginx/sites-enabled/default" \ TFW_NGINX_COMPONENTS="/etc/nginx/components" \ TFW_LIB_DIR="/usr/local/lib" \ - TFW_TERMINADO_DIR="/tmp/terminado_server" \ TFW_FRONTEND_DIR="/srv/frontend" \ - TFW_SERVER_DIR="/srv/.tfw" \ + TFW_DIR="/.tfw" \ + TFW_SERVER_DIR="/.tfw/tfw_server" \ + TFW_SNAPSHOTS_DIR="/.tfw/snapshots" \ TFW_AUTH_KEY="/tmp/tfw-auth.key" \ TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history" \ PROMPT_COMMAND="history -a" @@ -75,4 +76,4 @@ ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn install --frozen-lockfil ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn build --no-progress || : ONBUILD RUN test -z "${NOFRONTEND}" && mv /data/dist ${TFW_FRONTEND_DIR} && rm -rf /data || : -CMD exec supervisord --nodaemon +CMD exec supervisord --nodaemon --configuration ${TFW_SUPERVISORD_CONF} diff --git a/README.md b/README.md index d46cd4b..be9b156 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ Our pre-made event handlers are written in Python3, but you can write event hand This makes the framework really flexible: you can demonstrate the concepts you want to in any language while using the same set of tools provided by TFW. Inside Avatao this means that any of the content teams can use the framework with ease. +To implement an event handler in Python3 you should subclass the `EventHandlerBase` or `FSMAwareEventHandler` class in `tfw.event_handler_base` (the first provides a minimal working `EventHandler`, the second allows you to execute code on FSM events). + ### FSM Another unique feature of the framework is the FSM – finite state machine – representing the state of your challenge. @@ -74,20 +76,24 @@ The TFW message format: ```text { - "key: "some identifier used for addressing", + "key: ...some identifier used for addressing..., "data": { ... JSON object carrying anything, preferably cats ... }, - "trigger": "FSM action" + "trigger": ...FSM action..., + "signature": ...HMAC signature for authenticated messages..., + "seq": ...sequence number... } ``` - The `key` field is used by TFW for addressing and every message must have one (it can be an empty string though) - The `data` object can contain anything you might want to send - The `trigger` key is an optional field that triggers an FSM action with that name from the current state (whatever that might be) +- The `signature` field is present on authenticated messages (such as `fsm_update`s) +- The `seq` key is a counter incremented with each proxied message in the TFW server To mirror messages back to their sources you can use a special messaging format, in which the message to be mirrored is enveloped inside the `data` field of the outer message: @@ -117,6 +123,8 @@ APIs exposed by our pre-witten event handlers are documented here. ### IdeEventHandler +This event handler is responsible for reading and writing files shown in the frontend code editor. + You can read the content of the currently selected file like so: ``` { @@ -178,6 +186,15 @@ Overwriting the current list of excluded file patterns is possible with this mes ### TerminalEventHandler +Event handler responsible for running a backend for `xterm.js` to connect to (frontend terminal backend). + +By default callbacks on terminal history are invoked *as soon as* a command starts to execute in the terminal (they do not wait for the started command to finish, the callback may even run in paralell with the command). + +If you want to wait for them and invoke your callbacks *after* the command has finished, please set the `TFW_DELAY_HISTAPPEND` envvar to `1`. +Practically this can be done by appending an `export` to the user's `.bashrc` file from your `Dockerfile`, like so: + +`RUN echo "export TFW_DELAY_HISTAPPEND=1" >> /home/${AVATAO_USER}/.bashrc` + Writing to the terminal: ``` { @@ -204,6 +221,8 @@ You can read terminal command history like so: ### ProcessManagingEventHandler +This event handler is responsible for managing processes controlled by supervisord. + Starting, stopping and restarting supervisor processes can be done using similar messages (where `command` is `start`, `stop` or `restart`): ``` { @@ -218,6 +237,8 @@ Starting, stopping and restarting supervisor processes can be done using similar ### LogMonitoringEventHandler +Event handler emitting real time logs (`stdout` and `stderr`) from supervisord processes. + To change which supervisor process is monitored use this message: ``` { @@ -244,6 +265,8 @@ To set the tail length of logs (the monitor will send back the last `value` char ### FSMManagingEventHandler +This event handler controls the TFW finite state machine (FSM). + To attempt executing a trigger on the FSM use (this will also generate an FSM update message): ``` { @@ -279,3 +302,41 @@ This event handler broadcasts FSM update messages after handling commands in the } ``` +### DirectorySnapshottingEventHandler + +Event handler capable of taking and restoring snapshots of directories (saving and restoring directory contens). + +You can take a snapshot of the directories with the following message: +``` +{ + "key": "snapshot", + "data" : + { + "command": "take_snapshot" + } +} +``` + +To restore the state of the files in the directories use: +``` +{ + "key": "snapshot", + "data" : + { + "command": "restore_snapshot", + "value": ...date string (can parse ISO 8601, unix timestamp, etc.)... + } +} +``` + +It is also possible to exclude files that match given patterns (formatted like lines in `.gitignore` files): +``` +{ + "key": "snapshot", + "data" : + { + "command": "exclude", + "value": ...list of patterns to exclude from snapshots... + } +} +``` diff --git a/VERSION b/VERSION index c1385d5..4ed6a03 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -mainecoon +ocicat diff --git a/docs/source/components/components.rst b/docs/source/components/components.rst index 87dee5b..d2cef53 100644 --- a/docs/source/components/components.rst +++ b/docs/source/components/components.rst @@ -23,3 +23,10 @@ Components .. autoclass:: BashMonitor :members: + +.. autoclass:: FSMManagingEventHandler + :members: + +.. autoclass:: CommandsEqual + :members: + diff --git a/docs/source/foundations/eventhandlers.rst b/docs/source/foundations/eventhandlers.rst index 9b8d91c..be0a05e 100644 --- a/docs/source/foundations/eventhandlers.rst +++ b/docs/source/foundations/eventhandlers.rst @@ -3,10 +3,13 @@ Event handler base classes Subclass these to create your cusom event handlers. -.. automodule:: tfw +.. automodule:: tfw.event_handler_base .. autoclass:: EventHandlerBase :members: -.. autoclass:: TriggeredEventHandler +.. autoclass:: FSMAwareEventHandler + :members: + +.. autoclass:: BroadcastingEventHandler :members: diff --git a/docs/source/foundations/fsms.rst b/docs/source/foundations/fsms.rst index 190ee8b..ce50b31 100644 --- a/docs/source/foundations/fsms.rst +++ b/docs/source/foundations/fsms.rst @@ -3,10 +3,13 @@ FSM base classes Subclass these to create an FSM that fits your tutorial/challenge. -.. automodule:: tfw +.. automodule:: tfw.fsm .. autoclass:: FSMBase :members: .. autoclass:: LinearFSM :members: + +.. autoclass:: YamlFSM + :members: diff --git a/docs/source/index.rst b/docs/source/index.rst index 3d674ad..28b7a81 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -36,6 +36,16 @@ These are pre-written components for you to use, such as our IDE, terminal or co components/* +Utility +------- + +These are useful decorators, mixins and helpers to make common dev tasks easier. + +.. toctree:: + :glob: + + utility/* + Indices and tables ================== diff --git a/docs/source/networking/networking.rst b/docs/source/networking/networking.rst index 317d266..be18772 100644 --- a/docs/source/networking/networking.rst +++ b/docs/source/networking/networking.rst @@ -6,7 +6,7 @@ Networking .. autoclass:: TFWServerConnector :members: -.. automodule:: tfw.networking.event_handlers +.. automodule:: tfw.networking.event_handlers.server_connector .. autoclass:: ServerUplinkConnector :members: @@ -15,3 +15,8 @@ Networking .. autoclass:: MessageSender :members: + +.. automodule:: tfw.networking.fsm_aware + +.. autoclass:: FSMAware + :members: diff --git a/docs/source/utility/decorators.rst b/docs/source/utility/decorators.rst new file mode 100644 index 0000000..0e16c23 --- /dev/null +++ b/docs/source/utility/decorators.rst @@ -0,0 +1,10 @@ +TFW decorators +-------------- + +.. automodule:: tfw.decorators.rate_limiter + +.. autoclass:: RateLimiter + :members: + +.. autoclass:: AsyncRateLimiter + :members: diff --git a/lib/envvars/__init__.py b/lib/envvars/__init__.py index d1ed0d4..46774d4 100644 --- a/lib/envvars/__init__.py +++ b/lib/envvars/__init__.py @@ -4,7 +4,7 @@ from collections import namedtuple from os import environ -from tfw.decorators import lazy_property +from tfw.decorators.lazy_property import lazy_property class LazyEnvironment: diff --git a/lib/tfw/__init__.py b/lib/tfw/__init__.py index 04d934d..db64b25 100644 --- a/lib/tfw/__init__.py +++ b/lib/tfw/__init__.py @@ -1,7 +1,2 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. - -from .event_handler_base import EventHandlerBase, FSMAwareEventHandler, BroadcastingEventHandler -from .fsm_base import FSMBase -from .linear_fsm import LinearFSM -from .yaml_fsm import YamlFSM diff --git a/lib/tfw/components/__init__.py b/lib/tfw/components/__init__.py index 73c889d..7c3f76d 100644 --- a/lib/tfw/components/__init__.py +++ b/lib/tfw/components/__init__.py @@ -12,3 +12,5 @@ from .fsm_managing_event_handler import FSMManagingEventHandler from .snapshot_provider import SnapshotProvider from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler, PipeIOServer from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler +from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler +from .commands_equal import CommandsEqual diff --git a/lib/tfw/components/commands_equal.py b/lib/tfw/components/commands_equal.py new file mode 100644 index 0000000..d2d4d15 --- /dev/null +++ b/lib/tfw/components/commands_equal.py @@ -0,0 +1,110 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from shlex import split +from re import search + +from tfw.decorators.lazy_property import lazy_property + + +class CommandsEqual: + # pylint: disable=too-many-arguments + """ + This class is useful for comparing executed commands with + excepted commands (i.e. when triggering a state change when + the correct command is executed). + + Note that in most cases you should test the changes + caused by the commands instead of just checking command history + (stuff can be done in countless ways and preparing for every + single case is impossible). This should only be used when + testing the changes would be very difficult, like when + explaining stuff with cli tools and such. + + This class implicitly converts to bool, use it like + if CommandsEqual(...): ... + + It tries detecting differing command parameter orders with similar + semantics and provides fuzzy logic options. + The rationale behind this is that a few false positives + are better than only accepting a single version of a command + (i.e. using ==). + """ + def __init__( + self, command_1, command_2, + fuzzyness=1, begin_similarly=True, + include_patterns=None, exclude_patterns=None + ): + """ + :param command_1: Compared command 1 + :param command_2: Compared command 2 + :param fuzzyness: float between 0 and 1. + the percentage of arguments required to + match between commands to result in True. + i.e 1 means 100% - all arguments need to be + present in both commands, while 0.75 + would mean 75% - in case of 4 arguments + 1 could differ between the commands. + :param begin_similarly: bool, the first word of the commands + must match + :param include_patterns: list of regex patterns the commands + must include + :param exclude_patterns: list of regex patterns the commands + must exclude + """ + self.command_1 = split(command_1) + self.command_2 = split(command_2) + self.fuzzyness = fuzzyness + self.begin_similarly = begin_similarly + self.include_patterns = include_patterns + self.exclude_patterns = exclude_patterns + + def __bool__(self): + if self.begin_similarly: + if not self.beginnings_are_equal: + return False + + if self.include_patterns is not None: + if not self.commands_contain_include_patterns: + return False + + if self.exclude_patterns is not None: + if not self.commands_contain_no_exclude_patterns: + return False + + return self.similarity >= self.fuzzyness + + @lazy_property + def beginnings_are_equal(self): + return self.command_1[0] == self.command_2[0] + + @lazy_property + def commands_contain_include_patterns(self): + return all(( + self.contains_regex_patterns(self.command_1, self.include_patterns), + self.contains_regex_patterns(self.command_2, self.include_patterns) + )) + + @lazy_property + def commands_contain_no_exclude_patterns(self): + return all(( + not self.contains_regex_patterns(self.command_1, self.exclude_patterns), + not self.contains_regex_patterns(self.command_2, self.exclude_patterns) + )) + + @staticmethod + def contains_regex_patterns(command, regex_parts): + command = ' '.join(command) + for pattern in regex_parts: + if not search(pattern, command): + return False + return True + + @lazy_property + def similarity(self): + parts_1 = set(self.command_1) + parts_2 = set(self.command_2) + + difference = parts_1 - parts_2 + deviance = len(difference) / len(max(parts_1, parts_2)) + return 1 - deviance diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index c72a13e..f65c8ff 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -5,9 +5,9 @@ from functools import wraps from watchdog.events import FileSystemEventHandler as FileSystemWatchdogEventHandler -from tfw.networking.event_handlers import ServerUplinkConnector -from tfw.decorators import RateLimiter -from tfw.mixins import ObserverMixin +from tfw.networking.event_handlers.server_connector import ServerUplinkConnector +from tfw.decorators.rate_limiter import RateLimiter +from tfw.mixins.observer_mixin import ObserverMixin from tfw.config.logs import logging diff --git a/lib/tfw/components/directory_monitoring_event_handler.py b/lib/tfw/components/directory_monitoring_event_handler.py index 6f89e91..8d97022 100644 --- a/lib/tfw/components/directory_monitoring_event_handler.py +++ b/lib/tfw/components/directory_monitoring_event_handler.py @@ -3,10 +3,10 @@ from os.path import isdir, exists -from tfw import EventHandlerBase +from tfw.event_handler_base import EventHandlerBase +from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin +from tfw.components.directory_monitor import DirectoryMonitor from tfw.config.logs import logging -from tfw.mixins import MonitorManagerMixin -from .directory_monitor import DirectoryMonitor LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/directory_snapshotting_event_handler.py b/lib/tfw/components/directory_snapshotting_event_handler.py new file mode 100644 index 0000000..8b6f384 --- /dev/null +++ b/lib/tfw/components/directory_snapshotting_event_handler.py @@ -0,0 +1,87 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from os.path import join as joinpath +from os.path import basename +from os import makedirs +from datetime import datetime + +from dateutil import parser as dateparser + +from tfw.event_handler_base import EventHandlerBase +from tfw.components.snapshot_provider import SnapshotProvider +from tfw.config import TFWENV +from tfw.config.logs import logging + +LOG = logging.getLogger(__name__) + + +class DirectorySnapshottingEventHandler(EventHandlerBase): + def __init__(self, key, directories, exclude_unix_patterns=None): + super().__init__(key) + self.snapshot_providers = {} + self._exclude_unix_patterns = exclude_unix_patterns + self.init_snapshot_providers(directories) + + self.command_handlers = { + 'take_snapshot': self.handle_take_snapshot, + 'restore_snapshot': self.handle_restore_snapshot, + 'exclude': self.handle_exclude + } + + def init_snapshot_providers(self, directories): + for index, directory in enumerate(directories): + git_dir = self.init_git_dir(index, directory) + self.snapshot_providers[directory] = SnapshotProvider( + directory, + git_dir, + self._exclude_unix_patterns + ) + + @staticmethod + def init_git_dir(index, directory): + git_dir = joinpath( + TFWENV.SNAPSHOTS_DIR, + f'{basename(directory)}-{index}' + ) + makedirs(git_dir, exist_ok=True) + return git_dir + + def handle_event(self, message): + try: + data = message['data'] + message['data'] = self.command_handlers[data['command']](data) + return message + except KeyError: + LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) + + def handle_take_snapshot(self, data): + LOG.debug('Taking snapshots of directories %s', self.snapshot_providers.keys()) + for provider in self.snapshot_providers.values(): + provider.take_snapshot() + return data + + def handle_restore_snapshot(self, data): + date = dateparser.parse( + data.get( + 'value', + datetime.now().isoformat() + ) + ) + LOG.debug( + 'Restoring snapshots (@ %s) of directories %s', + date, + self.snapshot_providers.keys() + ) + for provider in self.snapshot_providers.values(): + provider.restore_snapshot(date) + return data + + def handle_exclude(self, data): + exclude_unix_patterns = data['value'] + if not isinstance(exclude_unix_patterns, list): + raise KeyError + + for provider in self.snapshot_providers.values(): + provider.exclude = exclude_unix_patterns + return data diff --git a/lib/tfw/components/fsm_managing_event_handler.py b/lib/tfw/components/fsm_managing_event_handler.py index 47761a0..73ebe10 100644 --- a/lib/tfw/components/fsm_managing_event_handler.py +++ b/lib/tfw/components/fsm_managing_event_handler.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw import EventHandlerBase +from tfw.event_handler_base import EventHandlerBase from tfw.crypto import KeyManager, sign_message, verify_message from tfw.config.logs import logging @@ -9,6 +9,20 @@ LOG = logging.getLogger(__name__) class FSMManagingEventHandler(EventHandlerBase): + """ + EventHandler responsible for managing the state machine of + the framework (TFW FSM). + + tfw.networking.TFWServer instances automatically send 'trigger' + commands to the event handler listening on the 'fsm' key, + which should be an instance of this event handler. + + This event handler accepts messages that have a + data['command'] key specifying a command to be executed. + + An 'fsm_update' message is broadcasted after every successful + command. + """ def __init__(self, key, fsm_type, require_signature=False): super().__init__(key) self.fsm = fsm_type() @@ -25,7 +39,7 @@ class FSMManagingEventHandler(EventHandlerBase): try: message = self.command_handlers[message['data']['command']](message) if message: - fsm_update_message = self._fsm_updater.generate_fsm_update() + fsm_update_message = self._fsm_updater.fsm_update sign_message(self.auth_key, message) sign_message(self.auth_key, fsm_update_message) self.server_connector.broadcast(fsm_update_message) @@ -34,6 +48,12 @@ class FSMManagingEventHandler(EventHandlerBase): LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) def handle_trigger(self, message): + """ + Attempts to step the FSM with the supplied trigger. + + :param message: TFW message with a data field containing + the action to try triggering in data['value'] + """ trigger = message['data']['value'] if self._require_signature: if not verify_message(self.auth_key, message): @@ -44,6 +64,9 @@ class FSMManagingEventHandler(EventHandlerBase): return None def handle_update(self, message): + """ + Does nothing, but triggers an 'fsm_update' message. + """ # pylint: disable=no-self-use return message @@ -52,23 +75,24 @@ class FSMUpdater: def __init__(self, fsm): self.fsm = fsm - def generate_fsm_update(self): + @property + def fsm_update(self): return { 'key': 'fsm_update', - 'data': self.get_fsm_state_and_transitions() + 'data': self.fsm_update_data } - def get_fsm_state_and_transitions(self): - state = self.fsm.state + @property + def fsm_update_data(self): valid_transitions = [ {'trigger': trigger} for trigger in self.fsm.get_triggers(self.fsm.state) ] - last_trigger = self.fsm.trigger_history[-1] if self.fsm.trigger_history else None - in_accepted_state = state in self.fsm.accepted_states + last_fsm_event = self.fsm.event_log[-1] + last_fsm_event['timestamp'] = last_fsm_event['timestamp'].isoformat() return { - 'current_state': state, + 'current_state': self.fsm.state, 'valid_transitions': valid_transitions, - 'last_trigger': last_trigger, - 'in_accepted_state': in_accepted_state + 'in_accepted_state': self.fsm.in_accepted_state, + 'last_event': last_fsm_event } diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index 9d259ae..410558a 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -8,8 +8,9 @@ from abc import ABC, abstractmethod from watchdog.events import PatternMatchingEventHandler -from tfw.mixins import CallbackMixin, ObserverMixin -from tfw.decorators import RateLimiter +from tfw.mixins.callback_mixin import CallbackMixin +from tfw.mixins.observer_mixin import ObserverMixin +from tfw.decorators.rate_limiter import RateLimiter class CallbackEventHandler(PatternMatchingEventHandler, ABC): diff --git a/lib/tfw/components/ide_event_handler.py b/lib/tfw/components/ide_event_handler.py index 2c698ad..7e242e5 100644 --- a/lib/tfw/components/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler.py @@ -6,10 +6,10 @@ from glob import glob from fnmatch import fnmatchcase from typing import Iterable -from tfw import EventHandlerBase -from tfw.mixins import MonitorManagerMixin +from tfw.event_handler_base import EventHandlerBase +from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin +from tfw.components.directory_monitor import DirectoryMonitor from tfw.config.logs import logging -from .directory_monitor import DirectoryMonitor LOG = logging.getLogger(__name__) @@ -157,7 +157,8 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): """ Read the currently selected file. - :return dict: message with the contents of the file in data['content'] + :return dict: TFW message data containing key 'content' + (contents of the selected file) """ try: data['content'] = self.filemanager.file_contents diff --git a/lib/tfw/components/log_monitor.py b/lib/tfw/components/log_monitor.py index 2c6ade8..cf06a0a 100644 --- a/lib/tfw/components/log_monitor.py +++ b/lib/tfw/components/log_monitor.py @@ -6,9 +6,10 @@ from os.path import dirname from watchdog.events import PatternMatchingEventHandler as PatternMatchingWatchdogEventHandler -from tfw.networking.event_handlers import ServerUplinkConnector -from tfw.decorators import RateLimiter -from tfw.mixins import ObserverMixin, SupervisorLogMixin +from tfw.networking.event_handlers.server_connector import ServerUplinkConnector +from tfw.decorators.rate_limiter import RateLimiter +from tfw.mixins.observer_mixin import ObserverMixin +from tfw.mixins.supervisor_mixin import SupervisorLogMixin class LogMonitor(ObserverMixin): diff --git a/lib/tfw/components/log_monitoring_event_handler.py b/lib/tfw/components/log_monitoring_event_handler.py index c5dd84c..0bc7ab2 100644 --- a/lib/tfw/components/log_monitoring_event_handler.py +++ b/lib/tfw/components/log_monitoring_event_handler.py @@ -1,10 +1,10 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw import EventHandlerBase -from tfw.mixins import MonitorManagerMixin +from tfw.event_handler_base import EventHandlerBase +from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin +from tfw.components.log_monitor import LogMonitor from tfw.config.logs import logging -from .log_monitor import LogMonitor LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/pipe_io_event_handler.py b/lib/tfw/components/pipe_io_event_handler.py index f9cd567..0a94e57 100644 --- a/lib/tfw/components/pipe_io_event_handler.py +++ b/lib/tfw/components/pipe_io_event_handler.py @@ -9,7 +9,7 @@ from secrets import token_urlsafe from threading import Thread from contextlib import suppress -from tfw import EventHandlerBase +from tfw.event_handler_base import EventHandlerBase from tfw.config.logs import logging from .pipe_io_server import PipeIOServer, terminate_process_on_failure diff --git a/lib/tfw/components/process_managing_event_handler.py b/lib/tfw/components/process_managing_event_handler.py index 040995e..61b6c62 100644 --- a/lib/tfw/components/process_managing_event_handler.py +++ b/lib/tfw/components/process_managing_event_handler.py @@ -3,10 +3,10 @@ from xmlrpc.client import Fault as SupervisorFault -from tfw import EventHandlerBase -from tfw.mixins import SupervisorMixin, SupervisorLogMixin +from tfw.event_handler_base import EventHandlerBase +from tfw.mixins.supervisor_mixin import SupervisorMixin, SupervisorLogMixin +from tfw.components.directory_monitor import with_monitor_paused from tfw.config.logs import logging -from .directory_monitor import with_monitor_paused LOG = logging.getLogger(__name__) diff --git a/lib/tfw/components/snapshot_provider.py b/lib/tfw/components/snapshot_provider.py index edee8ab..8bd17bf 100644 --- a/lib/tfw/components/snapshot_provider.py +++ b/lib/tfw/components/snapshot_provider.py @@ -2,15 +2,17 @@ # All Rights Reserved. See LICENSE file for details. import re -from subprocess import run, CalledProcessError +from subprocess import run, CalledProcessError, PIPE from getpass import getuser from os.path import isdir -from datetime import datetime +from os.path import join as joinpath from uuid import uuid4 +from dateutil import parser as dateparser + class SnapshotProvider: - def __init__(self, directory, git_dir): + def __init__(self, directory, git_dir, exclude_unix_patterns=None): self._classname = self.__class__.__name__ author = f'{getuser()} via TFW {self._classname}' self.gitenv = { @@ -25,6 +27,8 @@ class SnapshotProvider: self._init_repo() self.__last_valid_branch = self._branch + if exclude_unix_patterns: + self.exclude = exclude_unix_patterns def _init_repo(self): self._check_environment() @@ -66,10 +70,14 @@ class SnapshotProvider: 'git', 'add', '-A' )) - self._run(( - 'git', 'commit', - '-m', 'Snapshot' - )) + try: + self._get_stdout(( + 'git', 'commit', + '-m', 'Snapshot' + )) + except CalledProcessError as err: + if b'nothing to commit, working tree clean' not in err.output: + raise def _check_head_not_detached(self): if self._head_detached: @@ -87,7 +95,8 @@ class SnapshotProvider: )) def _get_stdout(self, *args, **kwargs): - kwargs['capture_output'] = True + kwargs['stdout'] = PIPE + kwargs['stderr'] = PIPE stdout_bytes = self._run(*args, **kwargs).stdout return stdout_bytes.decode().rstrip('\n') @@ -98,13 +107,31 @@ class SnapshotProvider: kwargs['env'] = self.gitenv return run(*args, **kwargs) + @property + def exclude(self): + with open(self._exclude_path, 'r') as ofile: + return ofile.read() + + @exclude.setter + def exclude(self, exclude_patterns): + with open(self._exclude_path, 'w') as ifile: + ifile.write('\n'.join(exclude_patterns)) + + @property + def _exclude_path(self): + return joinpath( + self.gitenv['GIT_DIR'], + 'info', + 'exclude' + ) + def take_snapshot(self): if self._head_detached: self._checkout_new_branch_from_head() self._snapshot() def _checkout_new_branch_from_head(self): - branch_name = uuid4() + branch_name = str(uuid4()) self._run(( 'git', 'branch', branch_name @@ -119,16 +146,30 @@ class SnapshotProvider: def restore_snapshot(self, date): commit = self._get_commit_from_timestamp(date) + branch = self._last_valid_branch + if commit == self._latest_commit_on_branch(branch): + commit = branch self._checkout(commit) def _get_commit_from_timestamp(self, date): - return self._get_stdout(( + commit = self._get_stdout(( 'git', 'rev-list', '--date=iso', '-n', '1', f'--before="{date.isoformat()}"', self._last_valid_branch )) + if not commit: + commit = self._get_oldest_parent_of_head() + return commit + + def _get_oldest_parent_of_head(self): + return self._get_stdout(( + 'git', + 'rev-list', + '--max-parents=0', + 'HEAD' + )) @property def _last_valid_branch(self): @@ -136,6 +177,14 @@ class SnapshotProvider: self.__last_valid_branch = self._branch return self.__last_valid_branch + def _latest_commit_on_branch(self, branch): + return self._get_stdout(( + 'git', 'log', + '-n', '1', + '--pretty=format:%H', + branch + )) + @property def all_timelines(self): return self._branches @@ -169,7 +218,7 @@ class SnapshotProvider: commit_hash, timestamp = line.split('@') commits.append({ 'hash': commit_hash, - 'timestamp': datetime.fromisoformat(timestamp) + 'timestamp': dateparser.parse(timestamp) }) return commits diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/components/terminal_event_handler.py index 79e96f7..7fbd74b 100644 --- a/lib/tfw/components/terminal_event_handler.py +++ b/lib/tfw/components/terminal_event_handler.py @@ -1,11 +1,11 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw import EventHandlerBase +from tfw.event_handler_base import EventHandlerBase +from tfw.components.terminado_mini_server import TerminadoMiniServer from tfw.config import TFWENV from tfw.config.logs import logging from tao.config import TAOENV -from .terminado_mini_server import TerminadoMiniServer LOG = logging.getLogger(__name__) @@ -26,7 +26,6 @@ class TerminalEventHandler(EventHandlerBase): :param monitor: tfw.components.HistoryMonitor instance to read command history from """ super().__init__(key) - self.working_directory = TFWENV.TERMINADO_DIR self._historymonitor = monitor bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash'] diff --git a/lib/tfw/crypto.py b/lib/tfw/crypto.py index af632d8..0b37893 100644 --- a/lib/tfw/crypto.py +++ b/lib/tfw/crypto.py @@ -14,8 +14,8 @@ from cryptography.hazmat.primitives.hashes import SHA256 from cryptography.hazmat.primitives.hmac import HMAC as _HMAC from cryptography.exceptions import InvalidSignature -from tfw.networking import message_bytes -from tfw.decorators import lazy_property +from tfw.networking.serialization import message_bytes +from tfw.decorators.lazy_property import lazy_property from tfw.config import TFWENV diff --git a/lib/tfw/decorators/__init__.py b/lib/tfw/decorators/__init__.py index ed79d7f..db64b25 100644 --- a/lib/tfw/decorators/__init__.py +++ b/lib/tfw/decorators/__init__.py @@ -1,5 +1,2 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. - -from .rate_limiter import RateLimiter -from .lazy_property import lazy_property diff --git a/lib/tfw/decorators/lazy_property.py b/lib/tfw/decorators/lazy_property.py index c7e00c6..14ad788 100644 --- a/lib/tfw/decorators/lazy_property.py +++ b/lib/tfw/decorators/lazy_property.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from functools import update_wrapper +from functools import update_wrapper, wraps class lazy_property: @@ -19,3 +19,12 @@ class lazy_property: value = self.func(instance) setattr(instance, self.func.__name__, value) return value + + +def lazy_factory(fun): + class wrapper: + @wraps(fun) + @lazy_property + def instance(self): # pylint: disable=no-self-use + return fun() + return wrapper() diff --git a/lib/tfw/decorators/rate_limiter.py b/lib/tfw/decorators/rate_limiter.py index abe6453..6f666c8 100644 --- a/lib/tfw/decorators/rate_limiter.py +++ b/lib/tfw/decorators/rate_limiter.py @@ -87,6 +87,7 @@ class AsyncRateLimiter(RateLimiter): return self._ioloop_factory() def action(self, seconds_to_next_call): + # pylint: disable=method-hidden if self._last_callback: self.ioloop.remove_timeout(self._last_callback) diff --git a/lib/tfw/event_handler_base/__init__.py b/lib/tfw/event_handler_base/__init__.py new file mode 100644 index 0000000..fd50525 --- /dev/null +++ b/lib/tfw/event_handler_base/__init__.py @@ -0,0 +1,6 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from .event_handler_base import EventHandlerBase +from .boradcasting_event_handler import BroadcastingEventHandler +from .fsm_aware_event_handler import FSMAwareEventHandler diff --git a/lib/tfw/event_handler_base/boradcasting_event_handler.py b/lib/tfw/event_handler_base/boradcasting_event_handler.py new file mode 100644 index 0000000..59ee493 --- /dev/null +++ b/lib/tfw/event_handler_base/boradcasting_event_handler.py @@ -0,0 +1,30 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from abc import ABC + +from tfw.event_handler_base.event_handler_base import EventHandlerBase +from tfw.crypto import message_checksum + + +class BroadcastingEventHandler(EventHandlerBase, ABC): + # pylint: disable=abstract-method + """ + Abstract base class for EventHandlers which broadcast responses + and intelligently ignore their own broadcasted messages they receive. + """ + def __init__(self, key): + super().__init__(key) + self.own_message_hashes = [] + + def event_handler_callback(self, message): + message_hash = message_checksum(message) + + if message_hash in self.own_message_hashes: + self.own_message_hashes.remove(message_hash) + return + + response = self.dispatch_handling(message) + if response: + self.own_message_hashes.append(message_checksum(response)) + self.server_connector.broadcast(response) diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base/event_handler_base.py similarity index 62% rename from lib/tfw/event_handler_base.py rename to lib/tfw/event_handler_base/event_handler_base.py index beb79a4..c013a8d 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base/event_handler_base.py @@ -5,8 +5,7 @@ from abc import ABC, abstractmethod from inspect import currentframe from typing import Iterable -from tfw.networking.event_handlers import ServerConnector -from tfw.crypto import message_checksum, KeyManager, verify_message +from tfw.networking.event_handlers.server_connector import ServerConnector from tfw.config.logs import logging LOG = logging.getLogger(__name__) @@ -125,64 +124,3 @@ class EventHandlerBase(ABC): instance for instance in locals_values if isinstance(instance, cls) } - - -class FSMAwareEventHandler(EventHandlerBase, ABC): - # pylint: disable=abstract-method - """ - Abstract base class for EventHandlers which automatically - keep track of the state of the TFW FSM. - """ - def __init__(self, key): - super().__init__(key) - self.subscribe('fsm_update') - self.fsm_state = None - self.in_accepted_state = False - self._auth_key = KeyManager().auth_key - - def dispatch_handling(self, message): - if message['key'] == 'fsm_update': - if verify_message(self._auth_key, message): - self._handle_fsm_update(message) - return None - return super().dispatch_handling(message) - - def _handle_fsm_update(self, message): - try: - new_state = message['data']['current_state'] - trigger = message['data']['last_trigger'] - if self.fsm_state != new_state: - self.handle_fsm_step(self.fsm_state, new_state, trigger) - self.fsm_state = new_state - self.in_accepted_state = message['data']['in_accepted_state'] - except KeyError: - LOG.error('Invalid fsm_update message received!') - - def handle_fsm_step(self, from_state, to_state, trigger): - """ - Called in case the TFW FSM has stepped. - """ - pass - - -class BroadcastingEventHandler(EventHandlerBase, ABC): - # pylint: disable=abstract-method - """ - Abstract base class for EventHandlers which broadcast responses - and intelligently ignore their own broadcasted messages they receive. - """ - def __init__(self, key): - super().__init__(key) - self.own_message_hashes = [] - - def event_handler_callback(self, message): - message_hash = message_checksum(message) - - if message_hash in self.own_message_hashes: - self.own_message_hashes.remove(message_hash) - return - - response = self.dispatch_handling(message) - if response: - self.own_message_hashes.append(message_checksum(response)) - self.server_connector.broadcast(response) diff --git a/lib/tfw/event_handler_base/fsm_aware_event_handler.py b/lib/tfw/event_handler_base/fsm_aware_event_handler.py new file mode 100644 index 0000000..01d6360 --- /dev/null +++ b/lib/tfw/event_handler_base/fsm_aware_event_handler.py @@ -0,0 +1,24 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from abc import ABC + +from tfw.event_handler_base.event_handler_base import EventHandlerBase +from tfw.networking.fsm_aware import FSMAware + + +class FSMAwareEventHandler(EventHandlerBase, FSMAware, ABC): + # pylint: disable=abstract-method + """ + Abstract base class for EventHandlers which automatically + keep track of the state of the TFW FSM. + """ + def __init__(self, key): + EventHandlerBase.__init__(self, key) + FSMAware.__init__(self) + self.subscribe('fsm_update') + + def dispatch_handling(self, message): + if self.update_fsm_data(message): + return None + return super().dispatch_handling(message) diff --git a/lib/tfw/fsm/__init__.py b/lib/tfw/fsm/__init__.py new file mode 100644 index 0000000..11a7a58 --- /dev/null +++ b/lib/tfw/fsm/__init__.py @@ -0,0 +1,6 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from .fsm_base import FSMBase +from .linear_fsm import LinearFSM +from .yaml_fsm import YamlFSM diff --git a/lib/tfw/fsm_base.py b/lib/tfw/fsm/fsm_base.py similarity index 72% rename from lib/tfw/fsm_base.py rename to lib/tfw/fsm/fsm_base.py index 7ae8157..d9fff29 100644 --- a/lib/tfw/fsm_base.py +++ b/lib/tfw/fsm/fsm_base.py @@ -2,10 +2,11 @@ # All Rights Reserved. See LICENSE file for details. from collections import defaultdict +from datetime import datetime from transitions import Machine, MachineError -from tfw.mixins import CallbackMixin +from tfw.mixins.callback_mixin import CallbackMixin from tfw.config.logs import logging LOG = logging.getLogger(__name__) @@ -22,9 +23,14 @@ class FSMBase(Machine, CallbackMixin): states, transitions = [], [] def __init__(self, initial=None, accepted_states=None): + """ + :param initial: which state to begin with, defaults to the last one + :param accepted_states: list of states in which the challenge should be + considered successfully completed + """ self.accepted_states = accepted_states or [self.states[-1].name] self.trigger_predicates = defaultdict(list) - self.trigger_history = [] + self.event_log = [] Machine.__init__( self, @@ -60,9 +66,22 @@ class FSMBase(Machine, CallbackMixin): if all(predicate_results): try: + from_state = self.state self.trigger(trigger) - self.trigger_history.append(trigger) + self.update_event_log(from_state, trigger) return True except (AttributeError, MachineError): LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger) return False + + def update_event_log(self, from_state, trigger): + self.event_log.append({ + 'from_state': from_state, + 'to_state': self.state, + 'trigger': trigger, + 'timestamp': datetime.utcnow() + }) + + @property + def in_accepted_state(self): + return self.state in self.accepted_states diff --git a/lib/tfw/linear_fsm.py b/lib/tfw/fsm/linear_fsm.py similarity index 78% rename from lib/tfw/linear_fsm.py rename to lib/tfw/fsm/linear_fsm.py index a053c30..5ac5eaf 100644 --- a/lib/tfw/linear_fsm.py +++ b/lib/tfw/fsm/linear_fsm.py @@ -3,7 +3,7 @@ from transitions import State -from .fsm_base import FSMBase +from tfw.fsm.fsm_base import FSMBase class LinearFSM(FSMBase): @@ -12,9 +12,13 @@ class LinearFSM(FSMBase): This is a state machine for challenges with linear progression, consisting of a number of steps specified in the constructor. It automatically sets up 2 actions (triggers) between states as such: - (0) -- step_1 --> (1) -- step_2 --> (2) -- step_3 --> (3) ... and so on + (0) -- step_1 --> (1) -- step_2 --> (2) -- step_3 --> (3) ... + (0) -- step_next --> (1) -- step_next --> (2) -- step_next --> (3) ... """ def __init__(self, number_of_steps): + """ + :param number_of_steps: how many states this FSM should have + """ self.states = [State(name=str(index)) for index in range(number_of_steps)] self.transitions = [] for state in self.states[:-1]: diff --git a/lib/tfw/yaml_fsm.py b/lib/tfw/fsm/yaml_fsm.py similarity index 83% rename from lib/tfw/yaml_fsm.py rename to lib/tfw/fsm/yaml_fsm.py index 63070d2..fbfc542 100644 --- a/lib/tfw/yaml_fsm.py +++ b/lib/tfw/fsm/yaml_fsm.py @@ -1,3 +1,6 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + from subprocess import Popen, run from functools import partial, singledispatch from contextlib import suppress @@ -6,11 +9,21 @@ import yaml import jinja2 from transitions import State -from tfw import FSMBase +from tfw.fsm.fsm_base import FSMBase class YamlFSM(FSMBase): + """ + This is a state machine capable of building itself from a YAML config file. + """ def __init__(self, config_file, jinja2_variables=None): + """ + :param config_file: path of the YAML file + :param jinja2_variables: dict containing jinja2 variables + or str with filename of YAML file to + parse and use as dict. + jinja2 support is disabled if this is None + """ self.config = ConfigParser(config_file, jinja2_variables).config self.setup_states() super().__init__() # FSMBase.__init__() requires states @@ -45,7 +58,7 @@ class YamlFSM(FSMBase): partial( command_statuscode_is_zero, predicate - ) + ) ) with suppress(KeyError): diff --git a/lib/tfw/mixins/__init__.py b/lib/tfw/mixins/__init__.py index 58915ca..db64b25 100644 --- a/lib/tfw/mixins/__init__.py +++ b/lib/tfw/mixins/__init__.py @@ -1,7 +1,2 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. - -from .supervisor_mixin import SupervisorMixin, SupervisorLogMixin -from .callback_mixin import CallbackMixin -from .observer_mixin import ObserverMixin -from .monitor_manager_mixin import MonitorManagerMixin diff --git a/lib/tfw/mixins/callback_mixin.py b/lib/tfw/mixins/callback_mixin.py index 33ddb6d..8075f47 100644 --- a/lib/tfw/mixins/callback_mixin.py +++ b/lib/tfw/mixins/callback_mixin.py @@ -3,7 +3,7 @@ from functools import partial -from tfw.decorators import lazy_property +from tfw.decorators.lazy_property import lazy_property class CallbackMixin: diff --git a/lib/tfw/mixins/observer_mixin.py b/lib/tfw/mixins/observer_mixin.py index 9d8b5e2..d5415e5 100644 --- a/lib/tfw/mixins/observer_mixin.py +++ b/lib/tfw/mixins/observer_mixin.py @@ -3,7 +3,7 @@ from watchdog.observers import Observer -from tfw.decorators import lazy_property +from tfw.decorators.lazy_property import lazy_property class ObserverMixin: diff --git a/lib/tfw/mixins/supervisor_mixin.py b/lib/tfw/mixins/supervisor_mixin.py index 2238985..189d7cc 100644 --- a/lib/tfw/mixins/supervisor_mixin.py +++ b/lib/tfw/mixins/supervisor_mixin.py @@ -6,7 +6,7 @@ from xmlrpc.client import Fault as SupervisorFault from contextlib import suppress from os import remove -from tfw.decorators import lazy_property +from tfw.decorators.lazy_property import lazy_property from tfw.config import TFWENV diff --git a/lib/tfw/networking/__init__.py b/lib/tfw/networking/__init__.py index c1f3eb4..500dd7a 100644 --- a/lib/tfw/networking/__init__.py +++ b/lib/tfw/networking/__init__.py @@ -1,9 +1,6 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from .serialization import serialize_tfw_msg, deserialize_tfw_msg -from .serialization import with_deserialize_tfw_msg, message_bytes -from .zmq_connector_base import ZMQConnectorBase from .message_sender import MessageSender from .event_handlers.server_connector import ServerUplinkConnector as TFWServerConnector from .server.tfw_server import TFWServer diff --git a/lib/tfw/networking/event_handlers/__init__.py b/lib/tfw/networking/event_handlers/__init__.py index b3ad530..db64b25 100644 --- a/lib/tfw/networking/event_handlers/__init__.py +++ b/lib/tfw/networking/event_handlers/__init__.py @@ -1,4 +1,2 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. - -from .server_connector import ServerConnector, ServerUplinkConnector, ServerDownlinkConnector diff --git a/lib/tfw/networking/event_handlers/server_connector.py b/lib/tfw/networking/event_handlers/server_connector.py index 612bb8d..7d24803 100644 --- a/lib/tfw/networking/event_handlers/server_connector.py +++ b/lib/tfw/networking/event_handlers/server_connector.py @@ -6,8 +6,8 @@ from functools import partial import zmq from zmq.eventloop.zmqstream import ZMQStream -from tfw.networking import serialize_tfw_msg, with_deserialize_tfw_msg -from tfw.networking import ZMQConnectorBase +from tfw.networking.zmq_connector_base import ZMQConnectorBase +from tfw.networking.serialization import serialize_tfw_msg, with_deserialize_tfw_msg from tfw.config import TFWENV from tfw.config.logs import logging diff --git a/lib/tfw/networking/fsm_aware.py b/lib/tfw/networking/fsm_aware.py new file mode 100644 index 0000000..cb2b287 --- /dev/null +++ b/lib/tfw/networking/fsm_aware.py @@ -0,0 +1,46 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + +from tfw.crypto import KeyManager, verify_message + +from tfw.config.logs import logging + +LOG = logging.getLogger(__name__) + + +class FSMAware: + """ + Base class for stuff that has to be aware of the framework FSM. + This is done by processing 'fsm_update' messages. + """ + def __init__(self): + self.fsm_state = None + self.fsm_in_accepted_state = False + self.fsm_event_log = [] + self._auth_key = KeyManager().auth_key + + def update_fsm_data(self, message): + if message['key'] == 'fsm_update' and verify_message(self._auth_key, message): + self._handle_fsm_update(message) + return True + return False + + def _handle_fsm_update(self, message): + try: + update_data = message['data'] + new_state = update_data['current_state'] + if self.fsm_state != new_state: + self.handle_fsm_step(**update_data) + self.fsm_state = new_state + self.fsm_in_accepted_state = update_data['in_accepted_state'] + self.fsm_event_log.append(update_data) + except KeyError: + LOG.error('Invalid fsm_update message received!') + + def handle_fsm_step(self, **kwargs): + """ + Called in case the TFW FSM has stepped. + + :param kwargs: fsm_update 'data' field + """ + pass diff --git a/lib/tfw/networking/message_sender.py b/lib/tfw/networking/message_sender.py index 378ad91..58eb1c9 100644 --- a/lib/tfw/networking/message_sender.py +++ b/lib/tfw/networking/message_sender.py @@ -1,7 +1,7 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from tfw.networking.event_handlers import ServerUplinkConnector +from tfw.networking.event_handlers.server_connector import ServerUplinkConnector class MessageSender: diff --git a/lib/tfw/networking/server/__init__.py b/lib/tfw/networking/server/__init__.py index eb9adde..db64b25 100644 --- a/lib/tfw/networking/server/__init__.py +++ b/lib/tfw/networking/server/__init__.py @@ -1,5 +1,2 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. - -from .event_handler_connector import EventHandlerConnector, EventHandlerUplinkConnector, EventHandlerDownlinkConnector -from .tfw_server import TFWServer diff --git a/lib/tfw/networking/server/event_handler_connector.py b/lib/tfw/networking/server/event_handler_connector.py index c4c6338..5075c56 100644 --- a/lib/tfw/networking/server/event_handler_connector.py +++ b/lib/tfw/networking/server/event_handler_connector.py @@ -4,7 +4,8 @@ import zmq from zmq.eventloop.zmqstream import ZMQStream -from tfw.networking import ZMQConnectorBase, serialize_tfw_msg, with_deserialize_tfw_msg +from tfw.networking.zmq_connector_base import ZMQConnectorBase +from tfw.networking.serialization import serialize_tfw_msg, with_deserialize_tfw_msg from tfw.config import TFWENV from tfw.config.logs import logging diff --git a/lib/tfw/networking/server/tfw_server.py b/lib/tfw/networking/server/tfw_server.py index c0d98f5..a1a0f5f 100644 --- a/lib/tfw/networking/server/tfw_server.py +++ b/lib/tfw/networking/server/tfw_server.py @@ -6,23 +6,25 @@ from contextlib import suppress from tornado.web import Application -from tfw.networking.event_handlers import ServerUplinkConnector -from tfw.networking.server import EventHandlerConnector -from tfw.networking import MessageSender +from tfw.networking.server.zmq_websocket_proxy import ZMQWebSocketProxy +from tfw.networking.event_handlers.server_connector import ServerUplinkConnector +from tfw.networking.server.event_handler_connector import EventHandlerConnector +from tfw.networking.message_sender import MessageSender +from tfw.networking.fsm_aware import FSMAware from tfw.crypto import KeyManager, verify_message, sign_message from tfw.config.logs import logging -from .zmq_websocket_proxy import ZMQWebSocketProxy LOG = logging.getLogger(__name__) -class TFWServer: +class TFWServer(FSMAware): """ This class handles the proxying of messages between the frontend and event handers. It proxies messages from the "/ws" route to all event handlers subscribed to a ZMQ SUB socket. """ def __init__(self): + super().__init__() self._event_handler_connector = EventHandlerConnector() self._uplink_connector = ServerUplinkConnector() self._auth_key = KeyManager().auth_key @@ -30,9 +32,16 @@ class TFWServer: self.application = Application([( r'/ws', ZMQWebSocketProxy, { 'event_handler_connector': self._event_handler_connector, - 'message_handlers': [self.handle_trigger, self.handle_recover], - 'frontend_message_handlers': [self.save_frontend_messages] - })]) + 'proxy_filters_and_callbacks': { + 'message_handlers': [ + self.handle_trigger, + self.handle_recover, + self.handle_fsm_update + ], + 'frontend_message_handlers': [self.save_frontend_messages] + } + } + )]) self._frontend_messages = FrontendMessageStorage() @@ -55,6 +64,9 @@ class TFWServer: self._frontend_messages.replay_messages(self._uplink_connector) self._frontend_messages.clear() + def handle_fsm_update(self, message): + self.update_fsm_data(message) + def save_frontend_messages(self, message): self._frontend_messages.save_message(message) diff --git a/lib/tfw/networking/server/zmq_websocket_proxy.py b/lib/tfw/networking/server/zmq_websocket_proxy.py index f456d94..15826bd 100644 --- a/lib/tfw/networking/server/zmq_websocket_proxy.py +++ b/lib/tfw/networking/server/zmq_websocket_proxy.py @@ -5,7 +5,7 @@ import json from tornado.websocket import WebSocketHandler -from tfw.mixins import CallbackMixin +from tfw.mixins.callback_mixin import CallbackMixin from tfw.config.logs import logging LOG = logging.getLogger(__name__) @@ -14,14 +14,11 @@ LOG = logging.getLogger(__name__) class ZMQWebSocketProxy(WebSocketHandler): # pylint: disable=abstract-method instances = set() + sequence_number = 0 def initialize(self, **kwargs): # pylint: disable=arguments-differ self._event_handler_connector = kwargs['event_handler_connector'] - - self._message_handlers = kwargs.get('message_handlers', []) - self._frontend_message_handlers = kwargs.get('frontend_message_handlers', []) - self._eventhandler_message_handlers = kwargs.get('eventhandler_message_handlers', []) - self._proxy_filters = kwargs.get('proxy_filters', []) + self._proxy_filters_and_callbacks = kwargs.get('proxy_filters_and_callbacks', {}) self.proxy_eventhandler_to_websocket = TFWProxy( self.send_eventhandler_message, @@ -35,14 +32,19 @@ class ZMQWebSocketProxy(WebSocketHandler): self.subscribe_proxy_callbacks() def subscribe_proxy_callbacks(self): + eventhandler_message_handlers = self._proxy_filters_and_callbacks.get('eventhandler_message_handlers', []) + frontend_message_handlers = self._proxy_filters_and_callbacks.get('frontend_message_handlers', []) + message_handlers = self._proxy_filters_and_callbacks.get('message_handlers', []) + proxy_filters = self._proxy_filters_and_callbacks.get('proxy_filters', []) + self.proxy_websocket_to_eventhandler.subscribe_proxy_callbacks_and_filters( - self._eventhandler_message_handlers + self._message_handlers, - self._proxy_filters + eventhandler_message_handlers + message_handlers, + proxy_filters ) self.proxy_eventhandler_to_websocket.subscribe_proxy_callbacks_and_filters( - self._frontend_message_handlers + self._message_handlers, - self._proxy_filters + frontend_message_handlers + message_handlers, + proxy_filters ) def prepare(self): @@ -59,14 +61,21 @@ class ZMQWebSocketProxy(WebSocketHandler): """ Invoked on ZMQ messages from event handlers. """ + self.sequence_message(message) LOG.debug('Received on pull socket: %s', message) self.proxy_eventhandler_to_websocket(message) + @classmethod + def sequence_message(cls, message): + cls.sequence_number += 1 + message['seq'] = cls.sequence_number + def on_message(self, message): """ Invoked on WS messages from frontend. """ message = json.loads(message) + self.sequence_message(message) LOG.debug('Received on WebSocket: %s', message) self.proxy_websocket_to_eventhandler(message) @@ -105,14 +114,9 @@ class TFWProxy: raise ValueError('Invalid TFW message format!') def __call__(self, message): - try: - self.proxy_filters._execute_callbacks(message) - except ValueError: - LOG.exception('Invalid TFW message received!') + if not self.filter_and_execute_callbacks(message): return - self.proxy_callbacks._execute_callbacks(message) - if message['key'] not in self.keyhandlers: self.to_destination(message) else: @@ -122,13 +126,26 @@ class TFWProxy: except KeyError: LOG.error('Invalid "%s" message format! Ignoring.', handler.__name__) + def filter_and_execute_callbacks(self, message): + try: + self.proxy_filters._execute_callbacks(message) + self.proxy_callbacks._execute_callbacks(message) + return True + except ValueError: + LOG.exception('Invalid TFW message received!') + return False + def mirror(self, message): message = message['data'] + if not self.filter_and_execute_callbacks(message): + return LOG.debug('Mirroring message: %s', message) self.to_source(message) def broadcast(self, message): message = message['data'] + if not self.filter_and_execute_callbacks(message): + return LOG.debug('Broadcasting message: %s', message) self.to_source(message) self.to_destination(message) diff --git a/requirements.txt b/requirements.txt index 2f133de..c2ab3ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,9 @@ -tornado==5.0 -pyzmq==17.0.0 -transitions==0.6.4 +tornado==5.1 +pyzmq==17.1.2 +transitions==0.6.6 terminado==0.8.1 watchdog==0.8.3 -PyYAML==3.12 +PyYAML==3.13 Jinja2==2.10 -cryptography==2.2.2 +cryptography==2.3.1 +python-dateutil==2.7.3