Merge pull request #38 from avatao-content/rate_limit_magic

Rate limit magic

.drone.yml

@@ -10,4 +10,4 @@ pipeline:
       - docker push eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG}
     when:
       event: 'tag'
-      branch: refs/tags/bombay-20*
+      branch: refs/tags/mainecoon-20*

Dockerfile

@@ -34,9 +34,11 @@ ENV PYTHONPATH="/usr/local/lib"                                            \
     TFW_NGINX_CONF="/etc/nginx/nginx.conf"                                 \
     TFW_NGINX_DEFAULT="/etc/nginx/sites-enabled/default"                   \
     TFW_NGINX_COMPONENTS="/etc/nginx/components"                           \
-    TFW_LIB_DIR="/usr/local/lib/"                                          \
+    TFW_LIB_DIR="/usr/local/lib"                                           \
     TFW_TERMINADO_DIR="/tmp/terminado_server"                              \
     TFW_FRONTEND_DIR="/srv/frontend"                                       \
+    TFW_SERVER_DIR="/srv/.tfw"                                             \
+    TFW_AUTH_KEY="/tmp/tfw-auth.key"                                       \
     TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history"                      \
     PROMPT_COMMAND="history -a"
 
@@ -45,13 +47,15 @@ RUN echo "export HISTFILE=${TFW_HISTFILE}" >> /tmp/bashrc &&\
     cat /tmp/bashrc >> /home/${AVATAO_USER}/.bashrc
 
 COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF}
+COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS}
 COPY nginx/nginx.conf ${TFW_NGINX_CONF}
 COPY nginx/default.conf ${TFW_NGINX_DEFAULT}
 COPY nginx/components/ ${TFW_NGINX_COMPONENTS}
-COPY lib LICENSE ${TFW_LIB_DIR}
+COPY lib LICENSE ${TFW_LIB_DIR}/
+COPY supervisor/tfw_server.py ${TFW_SERVER_DIR}/
 
 RUN for dir in "${TFW_LIB_DIR}"/{tfw,tao,envvars} "/etc/nginx" "/etc/supervisor"; do \
-        chown -R root:root "$dir" && chmod -R 700 "$dir";                               \
+        chown -R root:root "$dir" && chmod -R 700 "$dir";                            \
     done
 
 ONBUILD ARG BUILD_CONTEXT="solvable"
@@ -63,7 +67,7 @@ ONBUILD COPY ${BUILD_CONTEXT}/supervisor/ ${TFW_SUPERVISORD_COMPONENTS}
 ONBUILD RUN for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do                               \
                 envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f~ && mv $f~ $f ;\
             done
-ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx"]
+ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx", "${TFW_LIB_DIR}/envvars", "${TFW_LIB_DIR}/tfw"]
 
 ONBUILD COPY ${BUILD_CONTEXT}/frontend /data/
 ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn install --frozen-lockfile || :

README.md

@@ -20,6 +20,20 @@ Frontend components use websockets to connect to the TFW server, to which you ca
 
 ![TFW architecture](docs/tfw_architecture.png)
 
+### Networking details
+
+Event handlers connect to the TFW server using ZMQ.
+They receive messages on their `SUB`(scribe) sockets, which are connected to the `PUB`(lish) socket of the server.
+Event handlers reply on their `PUSH` socket, then their messages are received on the `PULL` socket of the server.
+
+The TFW server is basically just a fancy proxy.
+It's behaviour is quite simple: it proxies every message received from the fontend to the event handlers and vice versa.
+
+The server is also capable of "mirroring" messages back to their source.
+This is useful for communication between event handlers or frontend components (event handler to event handler or frontend component to frontend component communication).
+
+Components can also broadcast messages (broadcasted messages are received both by event handlers and the frontend as well).
+
 ### Event handlers
 
 Imagine event handlers as callbacks that are invoked when TFW receives a specific type of message. For instance, you could send a message to the framework when the user does something of note.
@@ -75,11 +89,193 @@ The TFW message format:
 - The `data` object can contain anything you might want to send
 - The `trigger` key is an optional field that triggers an FSM action with that name from the current state (whatever that might be)
 
+To mirror messages back to their sources you can use a special messaging format, in which the message to be mirrored is enveloped inside the `data` field of the outer message:
+
+```text
+    "key": "mirror",
+    "data":
+    {
+        ...
+        The message you want to mirror (with it's own "key" and "data" fields)
+        ...
+    }
+```
+
+Broadcasting messages is possible in a similar manner by using `"key": "broadcast"` in the outer message.
 
 ## Where to go next
 
 Most of the components you need have docstrings included (hang on tight, this is work in progress) – refer to them for usage info.
 
-In the `docs` folder you can find our Sphinx-based API documentation, which you can build using the included `Makefile` (you need to have Sphinx installed, please reach out to us if you have trouble building the docs).
+In the `docs` folder you can find our Sphinx-based documentation, which you can build using the `hack/tfw.sh` script in the [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework) repository.
+
+To get started you should take a look at [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework), which serves as an example project as well.
+
+## API
+
+APIs exposed by our pre-witten event handlers are documented here.
+
+### IdeEventHandler
+
+You can read the content of the currently selected file like so:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "read"
+    }
+}
+```
+
+Use the following message to overwrite the content of the currently selected file:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "write",
+        "content": ...string...
+    }
+}
+```
+
+To select a file use the following message:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "select",
+        "filename": ...string...
+    }
+}
+```
+
+You can switch to a new working directory using this message (note that the directory must be in `allowed_directories`):
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "selectdir",
+        "directory": ...string...
+    }
+}
+```
+
+Overwriting the current list of excluded file patterns is possible with this message:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "exclude",
+        "exclude": ...array of strings...
+    }
+}
+```
+
+### TerminalEventHandler
+
+Writing to the terminal:
+```
+{ 
+    "key": "shell",
+    "data":
+    {
+        "command": "write",
+        "value": ...string...
+    }
+}
+```
+
+You can read terminal command history like so:
+```
+{ 
+    "key": "shell",
+    "data":
+    {
+        "command": "read",
+        "count": ...number...
+    }
+}
+```
+
+### ProcessManagingEventHandler
+
+Starting, stopping and restarting supervisor processes can be done using similar messages (where `command` is `start`, `stop` or `restart`):
+```
+{ 
+    "key": "processmanager",
+    "data":
+    {
+        "command": ...string...,
+        "process_name": ...string...
+    }
+}
+```
+
+### LogMonitoringEventHandler
+
+To change which supervisor process is monitored use this message:
+```
+{ 
+    "key": "logmonitor",
+    "data" :
+    {
+        "command": "process_name",
+        "value": ...string...
+    }
+}
+```
+
+To set the tail length of logs (the monitor will send back the last `value` characters of the log):
+```
+{ 
+    "key": "logmonitor",
+    "data" :
+    {
+        "command": "log_tail",
+        "value": ...number...
+    }
+}
+```
+
+### FSMManagingEventHandler
+
+To attempt executing a trigger on the FSM use (this will also generate an FSM update message):
+```
+{
+    "key": "fsm",
+    "data" :
+    {
+        "command": "trigger",
+        "value": ...string...
+    }
+}
+```
+
+To force the broadcasting of an FSM update you can use this message:
+```
+{
+    "key": "fsm",
+    "data" :
+    {
+        "command": "update"
+    }
+}
+```
+
+This event handler broadcasts FSM update messages after handling commands in the following format:
+```
+{
+    "key": "fsm_update",
+    "data" :
+    {
+        "current_state": ...string...,
+        "valid_transitions": ...array of {"trigger": ...string...} objects...
+    }
+}
+```
 
-To get started you should take a look at the [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework) repository, which serves as an example project as well.

VERSION

@@ -1 +1 @@
-bombay
+mainecoon

bashrc

@@ -9,7 +9,7 @@ _precmd()
 {
     [ -z "$AT_PROMPT" ] && return
     unset AT_PROMPT
-    history -a
+    [ -z "$TFW_DELAY_HISTAPPEND" ] && history -a
 }
 trap _precmd DEBUG
 

docs/source/conf.py

@@ -26,7 +26,7 @@ author = 'Kristóf Tóth'
 # The short X.Y version
 version = ''
 # The full version, including alpha/beta/rc tags
-release = 'bombay'
+release = 'mainecoon'
 
 
 # -- General configuration ---------------------------------------------------

lib/tfw/__init__.py

@@ -1,6 +1,7 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from .event_handler_base import EventHandlerBase, TriggeredEventHandler
+from .event_handler_base import EventHandlerBase, FSMAwareEventHandler, BroadcastingEventHandler
 from .fsm_base import FSMBase
 from .linear_fsm import LinearFSM
+from .yaml_fsm import YamlFSM

lib/tfw/components/__init__.py

@@ -8,3 +8,5 @@ from .ide_event_handler import IdeEventHandler
 from .history_monitor import HistoryMonitor, BashMonitor, GDBMonitor
 from .terminal_commands import TerminalCommands
 from .log_monitoring_event_handler import LogMonitoringEventHandler
+from .fsm_managing_event_handler import FSMManagingEventHandler
+from .snapshot_provider import SnapshotProvider

lib/tfw/components/directory_monitor.py

@@ -15,8 +15,8 @@ LOG = logging.getLogger(__name__)
 
 
 class DirectoryMonitor(ObserverMixin):
-    def __init__(self, directories):
-        self.eventhandler = IdeReloadWatchdogEventHandler()
+    def __init__(self, ide_key, directories):
+        self.eventhandler = IdeReloadWatchdogEventHandler(ide_key)
         for directory in directories:
             self.observer.schedule(self.eventhandler, directory, recursive=True)
 
@@ -44,8 +44,9 @@ class DirectoryMonitor(ObserverMixin):
 
 
 class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler):
-    def __init__(self):
+    def __init__(self, ide_key):
         super().__init__()
+        self.ide_key = ide_key
         self.uplink = ServerUplinkConnector()
         self._paused = False
         self.ignore = 0
@@ -65,7 +66,7 @@ class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler):
             return
         LOG.debug(event)
         self.uplink.send({
-            'key': 'ide',
+            'key': self.ide_key,
             'data': {'command': 'reload'}
         })
 

lib/tfw/components/directory_monitoring_event_handler.py

@@ -15,7 +15,12 @@ class DirectoryMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin):
     def __init__(self, key, directory):
         super().__init__(key)
         self._directory = directory
-        MonitorManagerMixin.__init__(self, DirectoryMonitor, self._directory)
+        MonitorManagerMixin.__init__(
+            self,
+            DirectoryMonitor,
+            key,
+            self._directory
+        )
 
         self.commands = {
             'pause':     self.pause,

lib/tfw/components/fsm_managing_event_handler.py

@@ -0,0 +1,74 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from tfw import EventHandlerBase
+from tfw.crypto import KeyManager, sign_message, verify_message
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class FSMManagingEventHandler(EventHandlerBase):
+    def __init__(self, key, fsm_type, require_signature=False):
+        super().__init__(key)
+        self.fsm = fsm_type()
+        self._fsm_updater = FSMUpdater(self.fsm)
+        self.auth_key = KeyManager().auth_key
+        self._require_signature = require_signature
+
+        self.command_handlers = {
+            'trigger': self.handle_trigger,
+            'update':  self.handle_update
+        }
+
+    def handle_event(self, message):
+        try:
+            message = self.command_handlers[message['data']['command']](message)
+            if message:
+                fsm_update_message = self._fsm_updater.generate_fsm_update()
+                sign_message(self.auth_key, message)
+                sign_message(self.auth_key, fsm_update_message)
+                self.server_connector.broadcast(fsm_update_message)
+            return message
+        except KeyError:
+            LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
+
+    def handle_trigger(self, message):
+        trigger = message['data']['value']
+        if self._require_signature:
+            if not verify_message(self.auth_key, message):
+                LOG.error('Ignoring unsigned trigger command: %s', message)
+                return None
+        if self.fsm.step(trigger):
+            return message
+        return None
+
+    def handle_update(self, message):
+        # pylint: disable=no-self-use
+        return message
+
+
+class FSMUpdater:
+    def __init__(self, fsm):
+        self.fsm = fsm
+
+    def generate_fsm_update(self):
+        return {
+            'key': 'fsm_update',
+            'data': self.get_fsm_state_and_transitions()
+        }
+
+    def get_fsm_state_and_transitions(self):
+        state = self.fsm.state
+        valid_transitions = [
+            {'trigger': trigger}
+            for trigger in self.fsm.get_triggers(self.fsm.state)
+        ]
+        last_trigger = self.fsm.trigger_history[-1] if self.fsm.trigger_history else None
+        in_accepted_state = state in self.fsm.accepted_states
+        return {
+            'current_state': state,
+            'valid_transitions': valid_transitions,
+            'last_trigger': last_trigger,
+            'in_accepted_state': in_accepted_state
+        }

lib/tfw/components/ide_event_handler.py

@@ -4,7 +4,7 @@
 from os.path import isfile, join, relpath, exists, isdir, realpath
 from glob import glob
 from fnmatch import fnmatchcase
-from collections import Iterable
+from typing import Iterable
 
 from tfw import EventHandlerBase
 from tfw.mixins import MonitorManagerMixin
@@ -103,7 +103,7 @@ class FileManager:  # pylint: disable=too-many-instance-attributes
 
 
 class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
-    # pylint: disable=too-many-arguments
+    # pylint: disable=too-many-arguments,anomalous-backslash-in-string
     """
     Event handler implementing the backend of our browser based IDE.
     By default all files in the directory specified in __init__ are displayed
@@ -141,6 +141,7 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
         MonitorManagerMixin.__init__(
             self,
             DirectoryMonitor,
+            self.key,
             self.filemanager.allowed_directories
         )
 

lib/tfw/components/process_managing_event_handler.py

@@ -38,7 +38,6 @@ class ProcessManagingEventHandler(EventHandlerBase):
     """
     def __init__(self, key, dirmonitor=None, log_tail=0):
         super().__init__(key)
-        self.key = key
         self.monitor = dirmonitor
         self.processmanager = ProcessManager()
         self.log_tail = log_tail

lib/tfw/components/snapshot_provider.py

@@ -0,0 +1,175 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+import re
+from subprocess import run, CalledProcessError
+from getpass import getuser
+from os.path import isdir
+from datetime import datetime
+from uuid import uuid4
+
+
+class SnapshotProvider:
+    def __init__(self, directory, git_dir):
+        self._classname = self.__class__.__name__
+        author = f'{getuser()} via TFW {self._classname}'
+        self.gitenv = {
+            'GIT_DIR': git_dir,
+            'GIT_WORK_TREE': directory,
+            'GIT_AUTHOR_NAME': author,
+            'GIT_AUTHOR_EMAIL': '',
+            'GIT_COMMITTER_NAME': author,
+            'GIT_COMMITTER_EMAIL': '',
+            'GIT_PAGER': 'cat'
+        }
+
+        self._init_repo()
+        self.__last_valid_branch = self._branch
+
+    def _init_repo(self):
+        self._check_environment()
+
+        if not self._repo_is_initialized:
+            self._run(('git', 'init'))
+
+        if self._number_of_commits == 0:
+            try:
+                self._snapshot()
+            except CalledProcessError:
+                raise EnvironmentError(f'{self._classname} cannot init on empty directories!')
+
+        self._check_head_not_detached()
+
+    def _check_environment(self):
+        if not isdir(self.gitenv['GIT_DIR']) or not isdir(self.gitenv['GIT_WORK_TREE']):
+            raise EnvironmentError(f'{self._classname}: "directory" and "git_dir" must exist!')
+
+    @property
+    def _repo_is_initialized(self):
+        return self._run(
+            ('git', 'status'),
+            check=False
+        ).returncode == 0
+
+    @property
+    def _number_of_commits(self):
+        return int(
+            self._get_stdout((
+                'git', 'rev-list',
+                '--all',
+                '--count'
+            ))
+        )
+
+    def _snapshot(self):
+        self._run((
+            'git', 'add',
+            '-A'
+        ))
+        self._run((
+            'git', 'commit',
+            '-m', 'Snapshot'
+        ))
+
+    def _check_head_not_detached(self):
+        if self._head_detached:
+            raise EnvironmentError(f'{self._classname} cannot init from detached HEAD state!')
+
+    @property
+    def _head_detached(self):
+        return self._branch == 'HEAD'
+
+    @property
+    def _branch(self):
+        return self._get_stdout((
+            'git', 'rev-parse',
+            '--abbrev-ref', 'HEAD'
+        ))
+
+    def _get_stdout(self, *args, **kwargs):
+        kwargs['capture_output'] = True
+        stdout_bytes = self._run(*args, **kwargs).stdout
+        return stdout_bytes.decode().rstrip('\n')
+
+    def _run(self, *args, **kwargs):
+        if 'check' not in kwargs:
+            kwargs['check'] = True
+        if 'env' not in kwargs:
+            kwargs['env'] = self.gitenv
+        return run(*args, **kwargs)
+
+    def take_snapshot(self):
+        if self._head_detached:
+            self._checkout_new_branch_from_head()
+        self._snapshot()
+
+    def _checkout_new_branch_from_head(self):
+        branch_name = uuid4()
+        self._run((
+            'git', 'branch',
+            branch_name
+        ))
+        self._checkout(branch_name)
+
+    def _checkout(self, what):
+        self._run((
+            'git', 'checkout',
+            what
+        ))
+
+    def restore_snapshot(self, date):
+        commit = self._get_commit_from_timestamp(date)
+        self._checkout(commit)
+
+    def _get_commit_from_timestamp(self, date):
+        return self._get_stdout((
+            'git', 'rev-list',
+            '--date=iso',
+            '-n', '1',
+            f'--before="{date.isoformat()}"',
+            self._last_valid_branch
+        ))
+
+    @property
+    def _last_valid_branch(self):
+        if not self._head_detached:
+            self.__last_valid_branch = self._branch
+        return self.__last_valid_branch
+
+    @property
+    def all_timelines(self):
+        return self._branches
+
+    @property
+    def _branches(self):
+        git_branch_output = self._get_stdout(('git', 'branch'))
+        regex_pattern = re.compile(r'(?:[^\S\n]|[*])')  # matches '*' and non-newline whitespace chars
+        return re.sub(regex_pattern, '', git_branch_output).splitlines()
+
+    @property
+    def timeline(self):
+        return self._last_valid_branch
+
+    @timeline.setter
+    def timeline(self, value):
+        self._checkout(value)
+
+    @property
+    def snapshots(self):
+        return self._pretty_log_branch()
+
+    def _pretty_log_branch(self):
+        git_log_output = self._get_stdout((
+            'git', 'log',
+            '--pretty=%H@%aI'
+        ))
+
+        commits = []
+        for line in git_log_output.splitlines():
+            commit_hash, timestamp = line.split('@')
+            commits.append({
+                'hash': commit_hash,
+                'timestamp': datetime.fromisoformat(timestamp)
+            })
+
+        return commits

lib/tfw/components/terminal_commands.py

@@ -11,6 +11,7 @@ LOG = logging.getLogger(__name__)
 
 
 class TerminalCommands(ABC):
+    # pylint: disable=anomalous-backslash-in-string
     """
     A class you can use to define hooks for terminal commands. This means that you can
     have python code executed when the user enters a specific command to the terminal on

lib/tfw/components/terminal_event_handler.py

@@ -68,6 +68,7 @@ class TerminalEventHandler(EventHandlerBase):
                      (command to be written to the pty)
         """
         self.terminado_server.pty.write(data['value'])
+        return data
 
     def read(self, data):
         """

lib/tfw/crypto.py

@@ -0,0 +1,107 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from functools import wraps
+from base64 import b64encode, b64decode
+from copy import deepcopy
+from hashlib import md5
+from os import urandom, chmod
+from os.path import exists
+from stat import S_IRUSR, S_IWUSR, S_IXUSR
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.hashes import SHA256
+from cryptography.hazmat.primitives.hmac import HMAC as _HMAC
+from cryptography.exceptions import InvalidSignature
+
+from tfw.networking import message_bytes
+from tfw.decorators import lazy_property
+from tfw.config import TFWENV
+
+
+def message_checksum(message):
+    return md5(message_bytes(message)).hexdigest()
+
+
+def sign_message(key, message):
+    signature = message_signature(key, message)
+    message['signature'] = b64encode(signature).decode()
+
+
+def message_signature(key, message):
+    return HMAC(key, message_bytes(message)).signature
+
+
+def verify_message(key, message):
+    message = deepcopy(message)
+    try:
+        signature_b64 = message.pop('signature')
+        signature = b64decode(signature_b64)
+        actual_signature = message_signature(key, message)
+        return signature == actual_signature
+    except KeyError:
+        return False
+
+
+class KeyManager:
+    def __init__(self):
+        self.keyfile = TFWENV.AUTH_KEY
+        if not exists(self.keyfile):
+            self._init_auth_key()
+
+    @lazy_property
+    def auth_key(self):
+        with open(self.keyfile, 'rb') as ifile:
+            return ifile.read()
+
+    def _init_auth_key(self):
+        key = self.generate_key()
+        with open(self.keyfile, 'wb') as ofile:
+            ofile.write(key)
+        self._chmod_700_keyfile()
+        return key
+
+    @staticmethod
+    def generate_key():
+        return urandom(32)
+
+    def _chmod_700_keyfile(self):
+        chmod(self.keyfile, S_IRUSR | S_IWUSR | S_IXUSR)
+
+
+class HMAC:
+    def __init__(self, key, message):
+        self.key = key
+        self.message = message
+        self._hmac = _HMAC(
+            key=key,
+            algorithm=SHA256(),
+            backend=default_backend()
+        )
+
+    def _reload_if_finalized(f):
+        # pylint: disable=no-self-argument,not-callable
+        @wraps(f)
+        def wrapped(instance, *args, **kwargs):
+            if getattr(instance, '_finalized', False):
+                instance.__init__(instance.key, instance.message)
+            ret_val = f(instance, *args, **kwargs)
+            setattr(instance, '_finalized', True)
+            return ret_val
+        return wrapped
+
+    @property
+    @_reload_if_finalized
+    def signature(self):
+        self._hmac.update(self.message)
+        signature = self._hmac.finalize()
+        return signature
+
+    @_reload_if_finalized
+    def verify(self, signature):
+        self._hmac.update(self.message)
+        try:
+            self._hmac.verify(signature)
+            return True
+        except InvalidSignature:
+            return False

lib/tfw/decorators/rate_limiter.py

@@ -1,25 +1,100 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from functools import wraps
+from functools import wraps, partial
 from time import time, sleep
 
+from tfw.decorators.lazy_property import lazy_property
+
 
 class RateLimiter:
+    """
+    Decorator class for rate limiting, blocking.
+
+    When applied to a function this decorator will apply rate limiting
+    if the function is invoked more frequently than rate_per_seconds.
+
+    By default rate limiting means sleeping until the next invocation time
+    as per __init__ parameter rate_per_seconds.
+
+    Note that this decorator BLOCKS THE THREAD it is being executed on,
+    so it is only acceptable for stuff running on a separate thread.
+
+    If this is no good for you please refer to AsyncRateLimiter in this module,
+    which is designed not to block and use the IOLoop it is being called from.
+    """
     def __init__(self, rate_per_second):
+        """
+        :param rate_per_second: max frequency the decorated method should be
+                                invoked with
+        """
         self.min_interval = 1 / float(rate_per_second)
+        self.fun = None
         self.last_call = time()
 
+    def action(self, seconds_to_next_call):
+        if seconds_to_next_call:
+            sleep(seconds_to_next_call)
+        self.fun()
+
     def __call__(self, fun):
         @wraps(fun)
         def wrapper(*args, **kwargs):
-            self._limit_rate()
-            fun(*args, **kwargs)
+            self.fun = partial(fun, *args, **kwargs)
+            limit_seconds = self._limit_rate()
+            self.action(limit_seconds)
         return wrapper
 
     def _limit_rate(self):
-        since_last_call = time() - self.last_call
-        to_next_call = self.min_interval - since_last_call
+        seconds_since_last_call = time() - self.last_call
+        seconds_to_next_call = self.min_interval - seconds_since_last_call
+
+        if seconds_to_next_call > 0:
+            return seconds_to_next_call
         self.last_call = time()
-        if to_next_call > 0:
-            sleep(to_next_call)
+        return 0
+
+
+class AsyncRateLimiter(RateLimiter):
+    """
+    Decorator class for rate limiting, non-blocking.
+
+    The semantics of the rate limiting:
+        - unlike RateLimiter this decorator never blocks, instead it adds an async
+          callback version of the decorated function to the IOLoop
+          (to be executed after the rate limiting has expired).
+        - the timing works similarly to RateLimiter
+    """
+    def __init__(self, rate_per_second, ioloop_factory):
+        """
+        :param rate_per_second: max frequency the decorated method should be
+                                invoked with
+        :param ioloop_factory: callable that should return an instance of the
+                               IOLoop of the application
+        """
+        self._ioloop_factory = ioloop_factory
+        self._ioloop = None
+        self._last_callback = None
+
+        self._make_action_thread_safe()
+        super().__init__(rate_per_second=rate_per_second)
+
+    def _make_action_thread_safe(self):
+        self.action = partial(self.ioloop.add_callback, self.action)
+
+    @lazy_property
+    def ioloop(self):
+        return self._ioloop_factory()
+
+    def action(self, seconds_to_next_call):
+        if self._last_callback:
+            self.ioloop.remove_timeout(self._last_callback)
+
+        self._last_callback = self.ioloop.call_later(
+            seconds_to_next_call,
+            self.fun_with_debounce
+        )
+
+    def fun_with_debounce(self):
+        self.last_call = time()
+        self.fun()

lib/tfw/event_handler_base.py

@@ -3,8 +3,11 @@
 
 from abc import ABC, abstractmethod
 
-from tfw.networking import deserialize_tfw_msg
 from tfw.networking.event_handlers import ServerConnector
+from tfw.crypto import message_checksum, KeyManager, verify_message
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
 
 
 class EventHandlerBase(ABC):
@@ -16,30 +19,51 @@ class EventHandlerBase(ABC):
     """
     def __init__(self, key):
         self.server_connector = ServerConnector()
-        self.key = key
-        self.subscribe(self.key, 'reset')
+        self.keys = [key]
+        self.subscribe(*self.keys)
         self.server_connector.register_callback(self.event_handler_callback)
 
-    def event_handler_callback(self, msg_parts):
+    @property
+    def key(self):
+        """
+        Returns the oldest key this EventHandler was subscribed to.
+        """
+        return self.keys[0]
+
+    def event_handler_callback(self, message):
         """
         Callback that is invoked when receiving a message.
         Dispatches messages to handler methods and sends
         a response back in case the handler returned something.
         This is subscribed in __init__().
         """
-        message = deserialize_tfw_msg(*msg_parts)
+        if not self.check_key(message):
+            return
+
         response = self.dispatch_handling(message)
         if response:
-            response['key'] = message['key']
             self.server_connector.send(response)
 
+    def check_key(self, message):
+        """
+        Checks whether the message is intended for this
+        EventHandler.
+
+        This is necessary because ZMQ handles PUB - SUB
+        connetions with pattern matching (e.g. someone
+        subscribed to 'fsm' will receive 'fsm_update'
+        messages as well.
+        """
+        return message['key'] in self.keys
+
     def dispatch_handling(self, message):
         """
         Used to dispatch messages to their specific handlers.
+
+        :param message: the message received
+        :returns: the message to send back
         """
-        if message['key'] != 'reset':
-            return self.handle_event(message)
-        return self.handle_reset(message)
+        return self.handle_event(message)
 
     @abstractmethod
     def handle_event(self, message):
@@ -47,18 +71,10 @@ class EventHandlerBase(ABC):
         Abstract method that implements the handling of messages.
 
         :param message: the message received
+        :returns: the message to send back
         """
         raise NotImplementedError
 
-    def handle_reset(self, message):
-        # pylint: disable=unused-argument,no-self-use
-        """
-        Usually 'reset' events receive some sort of special treatment.
-
-        :param message: the message received
-        """
-        return None
-
     def subscribe(self, *keys):
         """
         Subscribe this EventHandler to receive events for given keys.
@@ -70,6 +86,7 @@ class EventHandlerBase(ABC):
         """
         for key in keys:
             self.server_connector.subscribe(key)
+            self.keys.append(key)
 
     def unsubscribe(self, *keys):
         """
@@ -79,6 +96,7 @@ class EventHandlerBase(ABC):
         """
         for key in keys:
             self.server_connector.unsubscribe(key)
+            self.keys.remove(key)
 
     def cleanup(self):
         """
@@ -88,17 +106,62 @@ class EventHandlerBase(ABC):
         pass
 
 
-class TriggeredEventHandler(EventHandlerBase, ABC):
+class FSMAwareEventHandler(EventHandlerBase, ABC):
     # pylint: disable=abstract-method
     """
-    Abstract base class for EventHandlers which are only triggered in case
-    TFWServer has successfully triggered an FSM step defined in __init__.
+    Abstract base class for EventHandlers which automatically
+    keep track of the state of the TFW FSM.
     """
-    def __init__(self, key, trigger):
+    def __init__(self, key):
         super().__init__(key)
-        self.trigger = trigger
+        self.subscribe('fsm_update')
+        self.fsm_state = None
+        self.in_accepted_state = False
+        self._auth_key = KeyManager().auth_key
 
     def dispatch_handling(self, message):
-        if message.get('trigger') == self.trigger:
-            return super().dispatch_handling(message)
-        return None
+        if message['key'] == 'fsm_update':
+            if verify_message(self._auth_key, message):
+                self._handle_fsm_update(message)
+            return None
+        return super().dispatch_handling(message)
+
+    def _handle_fsm_update(self, message):
+        try:
+            new_state = message['data']['current_state']
+            trigger = message['data']['last_trigger']
+            if self.fsm_state != new_state:
+                self.handle_fsm_step(self.fsm_state, new_state, trigger)
+            self.fsm_state = new_state
+            self.in_accepted_state = message['data']['in_accepted_state']
+        except KeyError:
+            LOG.error('Invalid fsm_update message received!')
+
+    def handle_fsm_step(self, from_state, to_state, trigger):
+        """
+        Called in case the TFW FSM has stepped.
+        """
+        pass
+
+
+class BroadcastingEventHandler(EventHandlerBase, ABC):
+    # pylint: disable=abstract-method
+    """
+    Abstract base class for EventHandlers which broadcast responses
+    and intelligently ignore their own broadcasted messages they receive.
+    """
+    def __init__(self, key):
+        super().__init__(key)
+        self.own_message_hashes = []
+
+    def event_handler_callback(self, message):
+        message_hash = message_checksum(message)
+
+        if message_hash in self.own_message_hashes:
+            self.own_message_hashes.remove(message_hash)
+            return
+
+        response = self.dispatch_handling(message)
+        if response:
+            self.own_message_hashes.append(message_checksum(response))
+            self.server_connector.broadcast(response)

lib/tfw/fsm_base.py

@@ -1,27 +1,33 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from typing import List
+from collections import defaultdict
 
-from transitions import Machine
+from transitions import Machine, MachineError
 
 from tfw.mixins import CallbackMixin
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
 
 
-class FSMBase(CallbackMixin):
+class FSMBase(Machine, CallbackMixin):
     """
     A general FSM base class you can inherit from to track user progress.
     See linear_fsm.py for an example use-case.
-    TFW the transitions library for state machines, please refer to their
+    TFW uses the transitions library for state machines, please refer to their
     documentation for more information on creating your own machines:
     https://github.com/pytransitions/transitions
     """
     states, transitions = [], []
 
-    def __init__(self, initial: str = None, accepted_states: List[str] = None):
-        self.accepted_states = accepted_states or [self.states[-1]]
-        self.machine = Machine(
-            model=self,
+    def __init__(self, initial=None, accepted_states=None):
+        self.accepted_states = accepted_states or [self.states[-1].name]
+        self.trigger_predicates = defaultdict(list)
+        self.trigger_history = []
+
+        Machine.__init__(
+            self,
             states=self.states,
             transitions=self.transitions,
             initial=initial or self.states[0],
@@ -34,4 +40,29 @@ class FSMBase(CallbackMixin):
         self._execute_callbacks(event_data.kwargs)
 
     def is_solved(self):
-        return self.state in self.accepted_states # pylint: disable=no-member
+        return self.state in self.accepted_states  # pylint: disable=no-member
+
+    def subscribe_predicate(self, trigger, *predicates):
+        self.trigger_predicates[trigger].extend(predicates)
+
+    def unsubscribe_predicate(self, trigger, *predicates):
+        self.trigger_predicates[trigger] = [
+            predicate
+            for predicate in self.trigger_predicates[trigger]
+            not in predicates
+        ]
+
+    def step(self, trigger):
+        predicate_results = (
+            predicate()
+            for predicate in self.trigger_predicates[trigger]
+        )
+
+        if all(predicate_results):
+            try:
+                self.trigger(trigger)
+                self.trigger_history.append(trigger)
+                return True
+            except (AttributeError, MachineError):
+                LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
+        return False

lib/tfw/linear_fsm.py

@@ -1,6 +1,8 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
+from transitions import State
+
 from .fsm_base import FSMBase
 
 
@@ -13,17 +15,17 @@ class LinearFSM(FSMBase):
     (0) --  step_1  --> (1) --  step_2  --> (2) --  step_3  --> (3) ... and so on
     """
     def __init__(self, number_of_steps):
-        self.states = list(map(str, range(number_of_steps)))
+        self.states = [State(name=str(index)) for index in range(number_of_steps)]
         self.transitions = []
-        for index in self.states[:-1]:
+        for state in self.states[:-1]:
             self.transitions.append({
-                'trigger': f'step_{int(index)+1}',
-                'source': index,
-                'dest': str(int(index)+1)
+                'trigger': f'step_{int(state.name)+1}',
+                'source': state.name,
+                'dest': str(int(state.name)+1)
             })
             self.transitions.append({
                 'trigger': 'step_next',
-                'source': index,
-                'dest': str(int(index)+1)
+                'source': state.name,
+                'dest': str(int(state.name)+1)
             })
         super(LinearFSM, self).__init__()

lib/tfw/mixins/callback_mixin.py

@@ -9,6 +9,7 @@ from tfw.decorators import lazy_property
 class CallbackMixin:
     @lazy_property
     def _callbacks(self):
+        # pylint: disable=no-self-use
         return []
 
     def subscribe_callback(self, callback, *args, **kwargs):
@@ -21,6 +22,14 @@ class CallbackMixin:
         fun = partial(callback, *args, **kwargs)
         self._callbacks.append(fun)
 
+    def subscribe_callbacks(self, *callbacks):
+        """
+        Subscribe a list of callbacks to incoke once an event is triggered.
+        :param callbacks: callbacks to be subscribed
+        """
+        for callback in callbacks:
+            self.subscribe_callback(callback)
+
     def unsubscribe_callback(self, callback):
         self._callbacks.remove(callback)
 

lib/tfw/mixins/observer_mixin.py

@@ -9,6 +9,7 @@ from tfw.decorators import lazy_property
 class ObserverMixin:
     @lazy_property
     def observer(self):
+        # pylint: disable=no-self-use
         return Observer()
 
     def watch(self):

lib/tfw/mixins/supervisor_mixin.py

@@ -13,6 +13,7 @@ from tfw.config import TFWENV
 class SupervisorBaseMixin:
     @lazy_property
     def supervisor(self):
+        # pylint: disable=no-self-use
         return xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor
 
 

lib/tfw/networking/__init__.py

@@ -1,9 +1,9 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from .serialization import serialize_tfw_msg, deserialize_tfw_msg, validate_message
+from .serialization import serialize_tfw_msg, deserialize_tfw_msg
+from .serialization import with_deserialize_tfw_msg, message_bytes
 from .zmq_connector_base import ZMQConnectorBase
-# from .controller_connector import ControllerConnector # TODO: readd once controller stuff is resolved
 from .message_sender import MessageSender
 from .event_handlers.server_connector import ServerUplinkConnector as TFWServerConnector
 from .server.tfw_server import TFWServer

lib/tfw/networking/controller_connector.py

@@ -1,18 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-import zmq
-from zmq.eventloop.zmqstream import ZMQStream
-
-from tfw.config import TFWENV
-from tfw.networking import ZMQConnectorBase
-
-
-class ControllerConnector(ZMQConnectorBase):
-    def __init__(self, zmq_context=None):
-        super(ControllerConnector, self).__init__(zmq_context)
-        self._zmq_rep_socket = self._zmq_context.socket(zmq.REP)
-        self._zmq_rep_socket.connect(f'tcp://localhost:{TFWENV.CONTROLLER_PORT}')
-        self._zmq_rep_stream = ZMQStream(self._zmq_rep_socket)
-
-        self.register_callback = self._zmq_rep_stream.on_recv_stream

lib/tfw/networking/event_handlers/server_connector.py

@@ -6,9 +6,12 @@ from functools import partial
 import zmq
 from zmq.eventloop.zmqstream import ZMQStream
 
-from tfw.networking import serialize_tfw_msg
+from tfw.networking import serialize_tfw_msg, with_deserialize_tfw_msg
 from tfw.networking import ZMQConnectorBase
 from tfw.config import TFWENV
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
 
 
 class ServerDownlinkConnector(ZMQConnectorBase):
@@ -20,7 +23,10 @@ class ServerDownlinkConnector(ZMQConnectorBase):
 
         self.subscribe = partial(self._zmq_sub_socket.setsockopt_string, zmq.SUBSCRIBE)
         self.unsubscribe = partial(self._zmq_sub_socket.setsockopt_string, zmq.UNSUBSCRIBE)
-        self.register_callback = self._zmq_sub_stream.on_recv
+
+    def register_callback(self, callback):
+        callback = with_deserialize_tfw_msg(callback)
+        self._zmq_sub_stream.on_recv(callback)
 
 
 class ServerUplinkConnector(ZMQConnectorBase):
@@ -34,26 +40,40 @@ class ServerUplinkConnector(ZMQConnectorBase):
 
     def send_to_eventhandler(self, message):
         """
-        Send a message to an event handler.
+        Send a message to an event handler through the TFW server.
+
         This envelopes the desired message in the 'data' field of the message to
         TFWServer, which will mirror it to event handlers.
 
         :param message: JSON message you want to send
-        :param message['key']: key of event handler you want to address
         """
-        nested_message = {'key': message['key'], 'data': message.pop('data')}
-        message['key'] = 'mirror'
-        message['data'] = nested_message
-        self.send(message)
+        self.send({
+            'key': 'mirror',
+            'data': message
+        })
 
     def send(self, message):
         """
-        Send a message to the TFW server
+        Send a message to the frontend through the TFW server.
 
         :param message: JSON message you want to send
         """
         self._zmq_push_socket.send_multipart(serialize_tfw_msg(message))
 
+    def broadcast(self, message):
+        """
+        Broadast a message through the TFW server.
+
+        This envelopes the desired message in the 'data' field of the message to
+        TFWServer, which will broadast it.
+
+        :param message: JSON message you want to send
+        """
+        self.send({
+            'key': 'broadcast',
+            'data': message
+        })
+
 
 class ServerConnector(ServerUplinkConnector, ServerDownlinkConnector):
     pass

lib/tfw/networking/message_sender.py

@@ -1,32 +1,54 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from datetime import datetime
-
 from tfw.networking.event_handlers import ServerUplinkConnector
 
 
 class MessageSender:
     """
-    Provides a mechanism to send messages to our frontend messaging component which
-    displays messages with the key "message".
+    Provides mechanisms to send messages to our frontend messaging component.
     """
-    def __init__(self, custom_key: str = None):
+    def __init__(self):
         self.server_connector = ServerUplinkConnector()
-        self.key = custom_key or 'message'
+        self.key = 'message'
+        self.queue_key = 'queueMessages'
 
     def send(self, originator, message):
         """
-        Sends a message to the key specified in __init__.
+        Sends a message.
         :param originator: name of sender to be displayed on the frontend
         :param message: message to send
         """
         data = {
             'originator': originator,
-            'timestamp': datetime.now().isoformat(),
             'message': message
         }
         self.server_connector.send({
             'key': self.key,
             'data': data
         })
+
+    def queue_messages(self, originator, messages):
+        """
+        Queues a list of messages to be displayed in a chatbot-like manner.
+        :param originator: name of sender to be displayed on the frontend
+        :param messages: list of messages to queue
+        """
+        data = {
+            'messages': [
+                {'message': message, 'originator': originator}
+                for message in messages
+            ]
+        }
+        self.server_connector.send({
+            'key': self.queue_key,
+            'data': data
+        })
+
+    @staticmethod
+    def generate_messages_from_queue(queue_message):
+        for message in queue_message['data']['messages']:
+            yield {
+                'key': 'message',
+                'data': message
+            }

lib/tfw/networking/serialization.py

@@ -22,10 +22,7 @@ The purpose of this module is abstracting away this low level behaviour.
 """
 
 import json
-
-
-def validate_message(message):
-    return 'key' in message
+from functools import wraps
 
 
 def serialize_tfw_msg(message):
@@ -35,6 +32,14 @@ def serialize_tfw_msg(message):
     return _serialize_all(message['key'], message)
 
 
+def with_deserialize_tfw_msg(fun):
+    @wraps(fun)
+    def wrapper(message_parts):
+        message = deserialize_tfw_msg(*message_parts)
+        return fun(message)
+    return wrapper
+
+
 def deserialize_tfw_msg(*args):
     """
     Return message from TFW multipart data
@@ -62,10 +67,14 @@ def _serialize_single(data):
     (serialize input if it is JSON)
     """
     if not isinstance(data, str):
-        data = json.dumps(data)
+        data = message_bytes(data)
     return _encode_if_needed(data)
 
 
+def message_bytes(message):
+    return json.dumps(message, sort_keys=True).encode()
+
+
 def _deserialize_single(data):
     """
     Try parsing input as JSON, return it as

lib/tfw/networking/server/__init__.py

@@ -3,5 +3,3 @@
 
 from .event_handler_connector import EventHandlerConnector, EventHandlerUplinkConnector, EventHandlerDownlinkConnector
 from .tfw_server import TFWServer
-from .zmq_websocket_handler import ZMQWebSocketProxy
-# from .controller_responder import ControllerResponder  # TODO: readd once controller stuff is resolved

lib/tfw/networking/server/controller_responder.py

@@ -1,38 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-from tfw.networking import deserialize_all, serialize_all, ControllerConnector
-
-
-class ControllerResponder:
-    def __init__(self, fsm):
-        self.fsm = fsm
-        self.token = None
-        self.controller_connector = ControllerConnector()
-        self.controller_connector.register_callback(self.handle_controller_request)
-        self.controller_request_handlers = {
-            'solution_check': self.handle_solution_check_request,
-            'test': self.handle_test_request,
-            'token': self.handle_token_request
-        }
-
-    def handle_controller_request(self, stream, msg_parts):
-        key, data = deserialize_all(*msg_parts)
-        response = self.controller_request_handlers[key](data)
-        stream.send_multipart(serialize_all(self.token, response))
-
-    def handle_test_request(self, data):
-        # pylint: disable=unused-argument,no-self-use
-        return 'OK'
-
-    def handle_token_request(self, data):
-        if self.token is None:
-            self.token = data
-        return {'token': self.token}
-
-    def handle_solution_check_request(self, data):
-        # pylint: disable=unused-argument
-        return {
-            'solved': self.fsm.is_solved(),
-            'message': 'solved' if self.fsm.is_solved() else 'not solved'
-        }

lib/tfw/networking/server/event_handler_connector.py

@@ -4,7 +4,7 @@
 import zmq
 from zmq.eventloop.zmqstream import ZMQStream
 
-from tfw.networking import ZMQConnectorBase, serialize_tfw_msg
+from tfw.networking import ZMQConnectorBase, serialize_tfw_msg, with_deserialize_tfw_msg
 from tfw.config import TFWENV
 from tfw.config.logs import logging
 
@@ -32,6 +32,7 @@ class EventHandlerUplinkConnector(ZMQConnectorBase):
 
 class EventHandlerConnector(EventHandlerDownlinkConnector, EventHandlerUplinkConnector):
     def register_callback(self, callback):
+        callback = with_deserialize_tfw_msg(callback)
         self._zmq_pull_stream.on_recv(callback)
 
     def send_message(self, message: dict):

lib/tfw/networking/server/tfw_server.py

@@ -1,15 +1,17 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from collections import defaultdict
+from abc import ABC, abstractmethod
+from contextlib import suppress
 
 from tornado.web import Application
 
-from tfw.networking import MessageSender
 from tfw.networking.event_handlers import ServerUplinkConnector
 from tfw.networking.server import EventHandlerConnector
+from tfw.networking import MessageSender
+from tfw.crypto import KeyManager, verify_message, sign_message
 from tfw.config.logs import logging
-from .zmq_websocket_handler import ZMQWebSocketProxy
+from .zmq_websocket_proxy import ZMQWebSocketProxy
 
 LOG = logging.getLogger(__name__)
 
@@ -18,117 +20,83 @@ class TFWServer:
     """
     This class handles the proxying of messages between the frontend and event handers.
     It proxies messages from the "/ws" route to all event handlers subscribed to a ZMQ
-    SUB socket. It also manages an FSM you can define as a constructor argument.
+    SUB socket.
     """
-    def __init__(self, fsm_type):
-        """
-        :param fsm_type: the type of FSM you want TFW to use
-        """
-        self._fsm = fsm_type()
-        self._fsm_updater = FSMUpdater(self._fsm)
-        self._fsm_manager = FSMManager(self._fsm)
-        self._fsm.subscribe_callback(self._fsm_updater.update)
+    def __init__(self):
         self._event_handler_connector = EventHandlerConnector()
+        self._uplink_connector = ServerUplinkConnector()
+        self._auth_key = KeyManager().auth_key
 
         self.application = Application([(
-            r'/ws', ZMQWebSocketProxy,{
-                'make_eventhandler_message': self.make_eventhandler_message,
-                'proxy_filter': self.proxy_filter,
-                'handle_trigger': self.handle_trigger,
-                'event_handler_connector': self._event_handler_connector
-            })]
-        )
-        # self.controller_responder = ControllerResponder(self.fsm)
-        # TODO: add this once controller stuff is resolved
+            r'/ws', ZMQWebSocketProxy, {
+                'event_handler_connector': self._event_handler_connector,
+                'message_handlers': [self.handle_trigger, self.handle_recover],
+                'frontend_message_handlers': [self.save_frontend_messages]
+            })])
 
-    @property
-    def fsm(self):
-        return self._fsm
-
-    @property
-    def fsm_manager(self):
-        return self._fsm_manager
-
-    def make_eventhandler_message(self, message):
-        self.trigger_fsm(message)
-        message['FSMUpdate'] = self._fsm_updater.get_fsm_state_and_transitions()
-        return message
+        self._frontend_messages = FrontendMessageStorage()
 
     def handle_trigger(self, message):
-        LOG.debug('Executing handler for trigger "%s"', message.get('trigger', ''))
-        self.trigger_fsm(message)
+        if 'trigger' in message:
+            LOG.debug('Executing handler for trigger "%s"', message.get('trigger', ''))
+            fsm_eh_command = {
+                'key': 'fsm',
+                'data': {
+                    'command': 'trigger',
+                    'value': message['trigger']
+                }
+            }
+            if verify_message(self._auth_key, message):
+                sign_message(self._auth_key, fsm_eh_command)
+            self._uplink_connector.send_to_eventhandler(fsm_eh_command)
 
-    def trigger_fsm(self, message):
-        trigger = message.get('trigger', '')
-        try:
-            self._fsm_manager.trigger(trigger, message)
-        except AttributeError:
-            LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
+    def handle_recover(self, message):
+        if message['key'] == 'recover':
+            self._frontend_messages.replay_messages(self._uplink_connector)
+            self._frontend_messages.clear()
 
-    def proxy_filter(self, message):
-        # pylint: disable=unused-argument,no-self-use
-        return True
+    def save_frontend_messages(self, message):
+        self._frontend_messages.save_message(message)
 
     def listen(self, port):
         self.application.listen(port)
 
 
-class FSMManager:
-    def __init__(self, fsm):
-        self._fsm = fsm
-        self.trigger_predicates = defaultdict(list)
-        self.messenge_sender = MessageSender()
+class MessageStorage(ABC):
+    def __init__(self):
+        self.saved_messages = []
 
-    @property
-    def fsm(self):
-        return self._fsm
+    def save_message(self, message):
+        with suppress(KeyError, AttributeError):
+            if self.filter_message(message):
+                self.saved_messages.extend(self.transform_message(message))
 
-    def trigger(self, trigger, message):
-        predicate_results = []
-        for predicate in self.trigger_predicates[trigger]:
-            success, message = predicate(message)
-            predicate_results.append(success)
-            self.messenge_sender.send('FSM', message)
+    @abstractmethod
+    def filter_message(self, message):
+        raise NotImplementedError
 
-        if all(predicate_results):
-            try:
-                self.fsm.trigger(trigger, message=message)
-            except AttributeError:
-                LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
+    def transform_message(self, message):   # pylint: disable=no-self-use
+        yield message
 
-    def subscribe_predicate(self, trigger, *predicates):
-        self.trigger_predicates[trigger].extend(predicates)
-
-    def unsubscribe_predicate(self, trigger, *predicates):
-        self.trigger_predicates[trigger] = [
-            predicate
-            for predicate in self.trigger_predicates[trigger]
-            not in predicates
-        ]
+    def clear(self):
+        self.saved_messages.clear()
 
 
-class FSMUpdater:
-    def __init__(self, fsm):
-        self.fsm = fsm
-        self.uplink = ServerUplinkConnector()
+class FrontendMessageStorage(MessageStorage):
+    def filter_message(self, message):
+        key = message['key']
+        command = message.get('data', {}).get('command')
+        return (
+            key in ('message', 'dashboard', 'queueMessages')
+            or key == 'ide' and command in ('select', 'read')
+        )
 
-    def update(self, kwargs_dict):
-        # pylint: disable=unused-argument
-        self.uplink.send(self.generate_fsm_update())
+    def transform_message(self, message):
+        if message['key'] == 'queueMessages':
+            yield from MessageSender.generate_messages_from_queue(message)
+        else:
+            yield message
 
-    def generate_fsm_update(self):
-        return {
-            'key': 'FSMUpdate',
-            'data': self.get_fsm_state_and_transitions()
-        }
-
-    def get_fsm_state_and_transitions(self):
-        state = self.fsm.state
-        valid_transitions = [
-            {'trigger': trigger}
-            for trigger in self.fsm.machine.get_triggers(self.fsm.state)
-        ]
-        return {
-            'current_state': state,
-            'valid_transitions': valid_transitions
-        }
+    def replay_messages(self, connector):
+        for message in self.saved_messages:
+            connector.send(message)

lib/tfw/networking/server/zmq_websocket_handler.py

@@ -1,91 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-import json
-from abc import ABC, abstractmethod
-
-from tornado.websocket import WebSocketHandler
-
-from tfw.networking import deserialize_tfw_msg, validate_message
-from tfw.config.logs import logging
-
-LOG = logging.getLogger(__name__)
-
-
-class ZMQWebSocketHandler(WebSocketHandler, ABC):
-    instances = set()
-
-    def initialize(self, **kwargs): # pylint: disable=arguments-differ
-        self._event_handler_connector = kwargs['event_handler_connector']
-
-    def prepare(self):
-        ZMQWebSocketHandler.instances.add(self)
-
-    def on_close(self):
-        ZMQWebSocketHandler.instances.remove(self)
-
-    def open(self, *args, **kwargs):
-        LOG.debug('WebSocket connection initiated')
-        self._event_handler_connector.register_callback(self.zmq_callback)
-
-    def zmq_callback(self, msg_parts):
-        keyhandlers = {'mirror': self.mirror}
-
-        message = deserialize_tfw_msg(*msg_parts)
-        LOG.debug('Received on pull socket: %s', message)
-        if not validate_message(message):
-            return
-
-        self.handle_trigger(message)
-        if message['key'] not in keyhandlers:
-            for instance in ZMQWebSocketHandler.instances:
-                instance.write_message(message)
-        else:
-            try:
-                keyhandlers[message['key']](message)
-            except KeyError:
-                LOG.error('Invalid mirror message format! Ignoring.')
-
-    def mirror(self, message):
-        message = message['data']
-        self._event_handler_connector.send_message(message)
-
-    def on_message(self, message):
-        LOG.debug('Received on WebSocket: %s', message)
-        if validate_message(message):
-            self.send_message(self.make_eventhandler_message(message))
-
-    @abstractmethod
-    def make_eventhandler_message(self, message):
-        raise NotImplementedError
-
-    def send_message(self, message: dict):
-        self._event_handler_connector.send_message(message)
-
-    @abstractmethod
-    def handle_trigger(self, message):
-        raise NotImplementedError
-
-    # much secure, very cors, wow
-    def check_origin(self, origin):
-        return True
-
-
-class ZMQWebSocketProxy(ZMQWebSocketHandler):
-    # pylint: disable=abstract-method
-    def initialize(self, **kwargs): # pylint: disable=arguments-differ
-        super(ZMQWebSocketProxy, self).initialize(**kwargs)
-        self._make_eventhandler_message = kwargs['make_eventhandler_message']
-        self._proxy_filter = kwargs['proxy_filter']
-        self._handle_trigger = kwargs['handle_trigger']
-
-    def on_message(self, message):
-        message = json.loads(message)
-        if self._proxy_filter(message):
-            super().on_message(message)
-
-    def make_eventhandler_message(self, message):
-        return self._make_eventhandler_message(message)
-
-    def handle_trigger(self, message):
-        self._handle_trigger(message)

lib/tfw/networking/server/zmq_websocket_proxy.py

@@ -0,0 +1,138 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+import json
+
+from tornado.websocket import WebSocketHandler
+
+from tfw.mixins import CallbackMixin
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class ZMQWebSocketProxy(WebSocketHandler):
+    # pylint: disable=abstract-method
+    instances = set()
+
+    def initialize(self, **kwargs): # pylint: disable=arguments-differ
+        self._event_handler_connector = kwargs['event_handler_connector']
+
+        self._message_handlers = kwargs.get('message_handlers', [])
+        self._frontend_message_handlers = kwargs.get('frontend_message_handlers', [])
+        self._eventhandler_message_handlers = kwargs.get('eventhandler_message_handlers', [])
+        self._proxy_filters = kwargs.get('proxy_filters', [])
+
+        self.proxy_eventhandler_to_websocket = TFWProxy(
+            self.send_eventhandler_message,
+            self.send_websocket_message
+        )
+        self.proxy_websocket_to_eventhandler = TFWProxy(
+            self.send_websocket_message,
+            self.send_eventhandler_message
+        )
+
+        self.subscribe_proxy_callbacks()
+
+    def subscribe_proxy_callbacks(self):
+        self.proxy_websocket_to_eventhandler.subscribe_proxy_callbacks_and_filters(
+            self._eventhandler_message_handlers + self._message_handlers,
+            self._proxy_filters
+        )
+
+        self.proxy_eventhandler_to_websocket.subscribe_proxy_callbacks_and_filters(
+            self._frontend_message_handlers + self._message_handlers,
+            self._proxy_filters
+        )
+
+    def prepare(self):
+        ZMQWebSocketProxy.instances.add(self)
+
+    def on_close(self):
+        ZMQWebSocketProxy.instances.remove(self)
+
+    def open(self, *args, **kwargs):
+        LOG.debug('WebSocket connection initiated')
+        self._event_handler_connector.register_callback(self.eventhander_callback)
+
+    def eventhander_callback(self, message):
+        """
+        Invoked on ZMQ messages from event handlers.
+        """
+        LOG.debug('Received on pull socket: %s', message)
+        self.proxy_eventhandler_to_websocket(message)
+
+    def on_message(self, message):
+        """
+        Invoked on WS messages from frontend.
+        """
+        message = json.loads(message)
+        LOG.debug('Received on WebSocket: %s', message)
+        self.proxy_websocket_to_eventhandler(message)
+
+    def send_eventhandler_message(self, message):
+        self._event_handler_connector.send_message(message)
+
+    @staticmethod
+    def send_websocket_message(message):
+        for instance in ZMQWebSocketProxy.instances:
+            instance.write_message(message)
+
+    # much secure, very cors, wow
+    def check_origin(self, origin):
+        return True
+
+
+class TFWProxy:
+    # pylint: disable=protected-access
+    def __init__(self, to_source, to_destination):
+        self.to_source = to_source
+        self.to_destination = to_destination
+
+        self.proxy_filters = CallbackMixin()
+        self.proxy_callbacks = CallbackMixin()
+
+        self.proxy_filters.subscribe_callback(self.validate_message)
+
+        self.keyhandlers = {
+            'mirror':    self.mirror,
+            'broadcast': self.broadcast
+        }
+
+    @staticmethod
+    def validate_message(message):
+        if 'key' not in message:
+            raise ValueError('Invalid TFW message format!')
+
+    def __call__(self, message):
+        try:
+            self.proxy_filters._execute_callbacks(message)
+        except ValueError:
+            LOG.exception('Invalid TFW message received!')
+            return
+
+        self.proxy_callbacks._execute_callbacks(message)
+
+        if message['key'] not in self.keyhandlers:
+            self.to_destination(message)
+        else:
+            handler = self.keyhandlers[message['key']]
+            try:
+                handler(message)
+            except KeyError:
+                LOG.error('Invalid "%s" message format! Ignoring.', handler.__name__)
+
+    def mirror(self, message):
+        message = message['data']
+        LOG.debug('Mirroring message: %s', message)
+        self.to_source(message)
+
+    def broadcast(self, message):
+        message = message['data']
+        LOG.debug('Broadcasting message: %s', message)
+        self.to_source(message)
+        self.to_destination(message)
+
+    def subscribe_proxy_callbacks_and_filters(self, proxy_callbacks, proxy_filters):
+        self.proxy_callbacks.subscribe_callbacks(*proxy_callbacks)
+        self.proxy_filters.subscribe_callbacks(*proxy_filters)

lib/tfw/yaml_fsm.py

@@ -0,0 +1,95 @@
+from subprocess import Popen, run
+from functools import partial, singledispatch
+from contextlib import suppress
+
+import yaml
+import jinja2
+from transitions import State
+
+from tfw import FSMBase
+
+
+class YamlFSM(FSMBase):
+    def __init__(self, config_file, jinja2_variables=None):
+        self.config = ConfigParser(config_file, jinja2_variables).config
+        self.setup_states()
+        super().__init__()  # FSMBase.__init__() requires states
+        self.setup_transitions()
+
+    def setup_states(self):
+        self.for_config_states_and_transitions_do(self.wrap_callbacks_with_subprocess_call)
+        self.states = [State(**state) for state in self.config['states']]
+
+    def setup_transitions(self):
+        self.for_config_states_and_transitions_do(self.subscribe_and_remove_predicates)
+        for transition in self.config['transitions']:
+            self.add_transition(**transition)
+
+    def for_config_states_and_transitions_do(self, what):
+        for array in ('states', 'transitions'):
+            for json_obj in self.config[array]:
+                what(json_obj)
+
+    @staticmethod
+    def wrap_callbacks_with_subprocess_call(json_obj):
+        topatch = ('on_enter', 'on_exit', 'prepare', 'before', 'after')
+        for key in json_obj:
+            if key in topatch:
+                json_obj[key] = partial(run_command_async, json_obj[key])
+
+    def subscribe_and_remove_predicates(self, json_obj):
+        if 'predicates' in json_obj:
+            for predicate in json_obj['predicates']:
+                self.subscribe_predicate(
+                    json_obj['trigger'],
+                    partial(
+                        command_statuscode_is_zero,
+                        predicate
+                        )
+                )
+
+        with suppress(KeyError):
+            json_obj.pop('predicates')
+
+
+def run_command_async(command, _):
+    Popen(command, shell=True)
+
+
+def command_statuscode_is_zero(command):
+    return run(command, shell=True).returncode == 0
+
+
+class ConfigParser:
+    def __init__(self, config_file, jinja2_variables):
+        self.read_variables = singledispatch(self._read_variables)
+        self.read_variables.register(dict, self._read_variables_dict)
+        self.read_variables.register(str, self._read_variables_str)
+
+        self.config = self.parse_config(config_file, jinja2_variables)
+
+    def parse_config(self, config_file, jinja2_variables):
+        config_string = self.read_file(config_file)
+        if jinja2_variables is not None:
+            variables = self.read_variables(jinja2_variables)
+            template = jinja2.Environment(loader=jinja2.BaseLoader).from_string(config_string)
+            config_string = template.render(**variables)
+        return yaml.safe_load(config_string)
+
+    @staticmethod
+    def read_file(filename):
+        with open(filename, 'r') as ifile:
+            return ifile.read()
+
+    @staticmethod
+    def _read_variables(variables):
+        raise TypeError(f'Invalid variables type {type(variables)}')
+
+    @staticmethod
+    def _read_variables_str(variables):
+        with open(variables, 'r') as ifile:
+            return yaml.safe_load(ifile)
+
+    @staticmethod
+    def _read_variables_dict(variables):
+        return variables

requirements.txt

@@ -3,3 +3,6 @@ pyzmq==17.0.0
 transitions==0.6.4
 terminado==0.8.1
 watchdog==0.8.3
+PyYAML==3.12
+Jinja2==2.10
+cryptography==2.2.2

setup.py

@@ -2,7 +2,6 @@ from os.path import dirname, realpath, join
 
 from setuptools import setup, find_packages
 
-
 here = dirname(realpath(__file__))
 
 with open(join(here, 'VERSION'), 'r') as ifile:
@@ -10,14 +9,21 @@ with open(join(here, 'VERSION'), 'r') as ifile:
 with open(join(here, 'requirements.txt'), 'r') as ifile:
     requirements = ifile.read().splitlines()
 
-setup(name='tfw',
-      version=version,
-      description='Avatao tutorial-framework',
-      url='https://github.com/avatao-content/baseimage-tutorial-framework',
-      author='Avatao.com Innovative Learning Kft.',
-      author_email='support@avatao.com',
-      license='custom',
-      packages=find_packages('lib'),
-      package_dir={'': 'lib'},
-      install_requires=requirements,
-      zip_safe=False)
+setup(
+    name = 'tfw',
+    version = version,
+    description = 'Avatao tutorial-framework',
+    url = 'https://github.com/avatao-content/baseimage-tutorial-framework',
+    author = 'Avatao.com Innovative Learning Kft.',
+    author_email = 'support@avatao.com',
+    license = 'custom',
+    packages = find_packages('lib'),
+    package_dir = {'': 'lib'},
+    install_requires = requirements,
+    extras_require = {
+        'docs': [
+            'sphinx >= 1.7.0',
+        ],
+    },
+    zip_safe = False,
+)

supervisor/components/tfw_server.conf

@@ -0,0 +1,4 @@
+[program:tfwserver]
+user=root
+directory=%(ENV_TFW_SERVER_DIR)s
+command=python3 tfw_server.py

supervisor/tfw_server.py

@@ -0,0 +1,9 @@
+from tornado.ioloop import IOLoop
+
+from tfw.networking import TFWServer
+from tfw.config import TFWENV
+
+
+if __name__ == '__main__':
+    TFWServer().listen(TFWENV.WEB_PORT)
+    IOLoop.instance().start()