Merge branch 'pipeio'

Rate limit magic

.drone.yml

@@ -1,6 +1,6 @@
 pipeline:
   build:
-    image: eu.gcr.io/avatao-public/docker:el7
+    image: eu.gcr.io/avatao-challengestore/challenge-builder
     volumes:
       - /etc/docker:/etc/docker:ro
       - /root/.docker:/root/.docker:ro
@@ -10,4 +10,4 @@ pipeline:
       - docker push eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG}
     when:
       event: 'tag'
-      branch: refs/tags/bombay-20*
+      branch: refs/tags/mainecoon-20*

Dockerfile

@@ -10,6 +10,7 @@ RUN curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash -
         supervisor                             \
         libzmq5                                \
         nginx                                  \
+        jq                                     \
         gettext-base                                                                                   &&\
     rm -rf /var/lib/apt/lists/*                                                                        &&\
     ln -sf /bin/bash /bin/sh
@@ -34,9 +35,11 @@ ENV PYTHONPATH="/usr/local/lib"                                            \
     TFW_NGINX_CONF="/etc/nginx/nginx.conf"                                 \
     TFW_NGINX_DEFAULT="/etc/nginx/sites-enabled/default"                   \
     TFW_NGINX_COMPONENTS="/etc/nginx/components"                           \
-    TFW_LIB_DIR="/usr/local/lib/"                                          \
+    TFW_LIB_DIR="/usr/local/lib"                                           \
     TFW_TERMINADO_DIR="/tmp/terminado_server"                              \
     TFW_FRONTEND_DIR="/srv/frontend"                                       \
+    TFW_SERVER_DIR="/srv/.tfw"                                             \
+    TFW_AUTH_KEY="/tmp/tfw-auth.key"                                       \
     TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history"                      \
     PROMPT_COMMAND="history -a"
 
@@ -45,10 +48,12 @@ RUN echo "export HISTFILE=${TFW_HISTFILE}" >> /tmp/bashrc &&\
     cat /tmp/bashrc >> /home/${AVATAO_USER}/.bashrc
 
 COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF}
+COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS}
 COPY nginx/nginx.conf ${TFW_NGINX_CONF}
 COPY nginx/default.conf ${TFW_NGINX_DEFAULT}
 COPY nginx/components/ ${TFW_NGINX_COMPONENTS}
-COPY lib LICENSE ${TFW_LIB_DIR}
+COPY lib LICENSE ${TFW_LIB_DIR}/
+COPY supervisor/tfw_server.py ${TFW_SERVER_DIR}/
 
 RUN for dir in "${TFW_LIB_DIR}"/{tfw,tao,envvars} "/etc/nginx" "/etc/supervisor"; do \
         chown -R root:root "$dir" && chmod -R 700 "$dir";                            \
@@ -63,7 +68,7 @@ ONBUILD COPY ${BUILD_CONTEXT}/supervisor/ ${TFW_SUPERVISORD_COMPONENTS}
 ONBUILD RUN for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do                               \
                 envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f~ && mv $f~ $f ;\
             done
-ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx"]
+ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx", "${TFW_LIB_DIR}/envvars", "${TFW_LIB_DIR}/tfw"]
 
 ONBUILD COPY ${BUILD_CONTEXT}/frontend /data/
 ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn install --frozen-lockfile || :

README.md

@@ -20,6 +20,20 @@ Frontend components use websockets to connect to the TFW server, to which you ca
 
 ![TFW architecture](docs/tfw_architecture.png)
 
+### Networking details
+
+Event handlers connect to the TFW server using ZMQ.
+They receive messages on their `SUB`(scribe) sockets, which are connected to the `PUB`(lish) socket of the server.
+Event handlers reply on their `PUSH` socket, then their messages are received on the `PULL` socket of the server.
+
+The TFW server is basically just a fancy proxy.
+It's behaviour is quite simple: it proxies every message received from the fontend to the event handlers and vice versa.
+
+The server is also capable of "mirroring" messages back to their source.
+This is useful for communication between event handlers or frontend components (event handler to event handler or frontend component to frontend component communication).
+
+Components can also broadcast messages (broadcasted messages are received both by event handlers and the frontend as well).
+
 ### Event handlers
 
 Imagine event handlers as callbacks that are invoked when TFW receives a specific type of message. For instance, you could send a message to the framework when the user does something of note.
@@ -75,11 +89,193 @@ The TFW message format:
 - The `data` object can contain anything you might want to send
 - The `trigger` key is an optional field that triggers an FSM action with that name from the current state (whatever that might be)
 
+To mirror messages back to their sources you can use a special messaging format, in which the message to be mirrored is enveloped inside the `data` field of the outer message:
+
+```text
+    "key": "mirror",
+    "data":
+    {
+        ...
+        The message you want to mirror (with it's own "key" and "data" fields)
+        ...
+    }
+```
+
+Broadcasting messages is possible in a similar manner by using `"key": "broadcast"` in the outer message.
 
 ## Where to go next
 
 Most of the components you need have docstrings included (hang on tight, this is work in progress) – refer to them for usage info.
 
-In the `docs` folder you can find our Sphinx-based API documentation, which you can build using the included `Makefile` (you need to have Sphinx installed, please reach out to us if you have trouble building the docs).
+In the `docs` folder you can find our Sphinx-based documentation, which you can build using the `hack/tfw.sh` script in the [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework) repository.
+
+To get started you should take a look at [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework), which serves as an example project as well.
+
+## API
+
+APIs exposed by our pre-witten event handlers are documented here.
+
+### IdeEventHandler
+
+You can read the content of the currently selected file like so:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "read"
+    }
+}
+```
+
+Use the following message to overwrite the content of the currently selected file:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "write",
+        "content": ...string...
+    }
+}
+```
+
+To select a file use the following message:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "select",
+        "filename": ...string...
+    }
+}
+```
+
+You can switch to a new working directory using this message (note that the directory must be in `allowed_directories`):
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "selectdir",
+        "directory": ...string...
+    }
+}
+```
+
+Overwriting the current list of excluded file patterns is possible with this message:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "exclude",
+        "exclude": ...array of strings...
+    }
+}
+```
+
+### TerminalEventHandler
+
+Writing to the terminal:
+```
+{ 
+    "key": "shell",
+    "data":
+    {
+        "command": "write",
+        "value": ...string...
+    }
+}
+```
+
+You can read terminal command history like so:
+```
+{ 
+    "key": "shell",
+    "data":
+    {
+        "command": "read",
+        "count": ...number...
+    }
+}
+```
+
+### ProcessManagingEventHandler
+
+Starting, stopping and restarting supervisor processes can be done using similar messages (where `command` is `start`, `stop` or `restart`):
+```
+{ 
+    "key": "processmanager",
+    "data":
+    {
+        "command": ...string...,
+        "process_name": ...string...
+    }
+}
+```
+
+### LogMonitoringEventHandler
+
+To change which supervisor process is monitored use this message:
+```
+{ 
+    "key": "logmonitor",
+    "data" :
+    {
+        "command": "process_name",
+        "value": ...string...
+    }
+}
+```
+
+To set the tail length of logs (the monitor will send back the last `value` characters of the log):
+```
+{ 
+    "key": "logmonitor",
+    "data" :
+    {
+        "command": "log_tail",
+        "value": ...number...
+    }
+}
+```
+
+### FSMManagingEventHandler
+
+To attempt executing a trigger on the FSM use (this will also generate an FSM update message):
+```
+{
+    "key": "fsm",
+    "data" :
+    {
+        "command": "trigger",
+        "value": ...string...
+    }
+}
+```
+
+To force the broadcasting of an FSM update you can use this message:
+```
+{
+    "key": "fsm",
+    "data" :
+    {
+        "command": "update"
+    }
+}
+```
+
+This event handler broadcasts FSM update messages after handling commands in the following format:
+```
+{
+    "key": "fsm_update",
+    "data" :
+    {
+        "current_state": ...string...,
+        "valid_transitions": ...array of {"trigger": ...string...} objects...
+    }
+}
+```
 
-To get started you should take a look at the [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework) repository, which serves as an example project as well.

VERSION

@@ -1 +1 @@
-bombay
+mainecoon

bashrc

@@ -9,7 +9,7 @@ _precmd()
 {
     [ -z "$AT_PROMPT" ] && return
     unset AT_PROMPT
-    history -a
+    [ -z "$TFW_DELAY_HISTAPPEND" ] && history -a
 }
 trap _precmd DEBUG
 

docs/source/conf.py

@@ -26,7 +26,7 @@ author = 'Kristóf Tóth'
 # The short X.Y version
 version = ''
 # The full version, including alpha/beta/rc tags
-release = 'bombay'
+release = 'mainecoon'
 
 
 # -- General configuration ---------------------------------------------------

lib/tfw/__init__.py

@@ -1,6 +1,7 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from .event_handler_base import EventHandlerBase, TriggeredEventHandler
+from .event_handler_base import EventHandlerBase, FSMAwareEventHandler, BroadcastingEventHandler
 from .fsm_base import FSMBase
 from .linear_fsm import LinearFSM
+from .yaml_fsm import YamlFSM

lib/tfw/components/__init__.py

@@ -8,3 +8,7 @@ from .ide_event_handler import IdeEventHandler
 from .history_monitor import HistoryMonitor, BashMonitor, GDBMonitor
 from .terminal_commands import TerminalCommands
 from .log_monitoring_event_handler import LogMonitoringEventHandler
+from .fsm_managing_event_handler import FSMManagingEventHandler
+from .snapshot_provider import SnapshotProvider
+from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler, PipeIOServer
+from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler

lib/tfw/components/directory_monitor.py

@@ -15,8 +15,8 @@ LOG = logging.getLogger(__name__)
 
 
 class DirectoryMonitor(ObserverMixin):
-    def __init__(self, directories):
-        self.eventhandler = IdeReloadWatchdogEventHandler()
+    def __init__(self, ide_key, directories):
+        self.eventhandler = IdeReloadWatchdogEventHandler(ide_key)
         for directory in directories:
             self.observer.schedule(self.eventhandler, directory, recursive=True)
 
@@ -44,8 +44,9 @@ class DirectoryMonitor(ObserverMixin):
 
 
 class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler):
-    def __init__(self):
+    def __init__(self, ide_key):
         super().__init__()
+        self.ide_key = ide_key
         self.uplink = ServerUplinkConnector()
         self._paused = False
         self.ignore = 0
@@ -65,7 +66,7 @@ class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler):
             return
         LOG.debug(event)
         self.uplink.send({
-            'key': 'ide',
+            'key': self.ide_key,
             'data': {'command': 'reload'}
         })
 

lib/tfw/components/directory_monitoring_event_handler.py

@@ -15,7 +15,12 @@ class DirectoryMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin):
     def __init__(self, key, directory):
         super().__init__(key)
         self._directory = directory
-        MonitorManagerMixin.__init__(self, DirectoryMonitor, self._directory)
+        MonitorManagerMixin.__init__(
+            self,
+            DirectoryMonitor,
+            key,
+            self._directory
+        )
 
         self.commands = {
             'pause':     self.pause,

lib/tfw/components/fsm_managing_event_handler.py

@@ -0,0 +1,74 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from tfw import EventHandlerBase
+from tfw.crypto import KeyManager, sign_message, verify_message
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class FSMManagingEventHandler(EventHandlerBase):
+    def __init__(self, key, fsm_type, require_signature=False):
+        super().__init__(key)
+        self.fsm = fsm_type()
+        self._fsm_updater = FSMUpdater(self.fsm)
+        self.auth_key = KeyManager().auth_key
+        self._require_signature = require_signature
+
+        self.command_handlers = {
+            'trigger': self.handle_trigger,
+            'update':  self.handle_update
+        }
+
+    def handle_event(self, message):
+        try:
+            message = self.command_handlers[message['data']['command']](message)
+            if message:
+                fsm_update_message = self._fsm_updater.generate_fsm_update()
+                sign_message(self.auth_key, message)
+                sign_message(self.auth_key, fsm_update_message)
+                self.server_connector.broadcast(fsm_update_message)
+            return message
+        except KeyError:
+            LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
+
+    def handle_trigger(self, message):
+        trigger = message['data']['value']
+        if self._require_signature:
+            if not verify_message(self.auth_key, message):
+                LOG.error('Ignoring unsigned trigger command: %s', message)
+                return None
+        if self.fsm.step(trigger):
+            return message
+        return None
+
+    def handle_update(self, message):
+        # pylint: disable=no-self-use
+        return message
+
+
+class FSMUpdater:
+    def __init__(self, fsm):
+        self.fsm = fsm
+
+    def generate_fsm_update(self):
+        return {
+            'key': 'fsm_update',
+            'data': self.get_fsm_state_and_transitions()
+        }
+
+    def get_fsm_state_and_transitions(self):
+        state = self.fsm.state
+        valid_transitions = [
+            {'trigger': trigger}
+            for trigger in self.fsm.get_triggers(self.fsm.state)
+        ]
+        last_trigger = self.fsm.trigger_history[-1] if self.fsm.trigger_history else None
+        in_accepted_state = state in self.fsm.accepted_states
+        return {
+            'current_state': state,
+            'valid_transitions': valid_transitions,
+            'last_trigger': last_trigger,
+            'in_accepted_state': in_accepted_state
+        }

lib/tfw/components/ide_event_handler.py

@@ -4,7 +4,7 @@
 from os.path import isfile, join, relpath, exists, isdir, realpath
 from glob import glob
 from fnmatch import fnmatchcase
-from collections import Iterable
+from typing import Iterable
 
 from tfw import EventHandlerBase
 from tfw.mixins import MonitorManagerMixin
@@ -103,7 +103,7 @@ class FileManager:  # pylint: disable=too-many-instance-attributes
 
 
 class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
-    # pylint: disable=too-many-arguments
+    # pylint: disable=too-many-arguments,anomalous-backslash-in-string
     """
     Event handler implementing the backend of our browser based IDE.
     By default all files in the directory specified in __init__ are displayed
@@ -141,6 +141,7 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
         MonitorManagerMixin.__init__(
             self,
             DirectoryMonitor,
+            self.key,
             self.filemanager.allowed_directories
         )
 

lib/tfw/components/pipe_io_event_handler.py

@@ -0,0 +1,143 @@
+from abc import abstractmethod
+from json import loads, dumps
+from subprocess import run, PIPE, Popen
+from functools import partial
+from os import getpgid, killpg
+from os.path import join
+from signal import SIGTERM
+from secrets import token_urlsafe
+from threading import Thread
+from contextlib import suppress
+
+from tfw import EventHandlerBase
+from tfw.config.logs import logging
+
+from .pipe_io_server import PipeIOServer, terminate_process_on_failure
+
+LOG = logging.getLogger(__name__)
+DEFAULT_PERMISSIONS = 0o600
+
+
+class PipeIOEventHandlerBase(EventHandlerBase):
+    def __init__(self, key, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS):
+        super().__init__(key)
+        self.pipe_io = CallbackPipeIOServer(
+            in_pipe_path,
+            out_pipe_path,
+            self.handle_pipe_event,
+            permissions
+        )
+        self.pipe_io.start()
+
+    @abstractmethod
+    def handle_pipe_event(self, message_bytes):
+        raise NotImplementedError()
+
+    def cleanup(self):
+        self.pipe_io.stop()
+
+
+class CallbackPipeIOServer(PipeIOServer):
+    def __init__(self, in_pipe_path, out_pipe_path, callback, permissions):
+        super().__init__(in_pipe_path, out_pipe_path, permissions)
+        self.callback = callback
+
+    def handle_message(self, message):
+        try:
+            self.callback(message)
+        except:  # pylint: disable=bare-except
+            LOG.exception('Failed to handle message %s from pipe %s!', message, self.in_pipe)
+
+
+class PipeIOEventHandler(PipeIOEventHandlerBase):
+    def handle_event(self, message):
+        json_bytes = dumps(message).encode()
+        self.pipe_io.send_message(json_bytes)
+
+    def handle_pipe_event(self, message_bytes):
+        json = loads(message_bytes)
+        self.server_connector.send(json)
+
+
+class TransformerPipeIOEventHandler(PipeIOEventHandlerBase):
+    # pylint: disable=too-many-arguments
+    def __init__(
+            self, key, in_pipe_path, out_pipe_path,
+            transform_in_cmd, transform_out_cmd,
+            permissions=DEFAULT_PERMISSIONS
+    ):
+        self._transform_in = partial(self._transform_message, transform_in_cmd)
+        self._transform_out = partial(self._transform_message, transform_out_cmd)
+        super().__init__(key, in_pipe_path, out_pipe_path, permissions)
+
+    @staticmethod
+    def _transform_message(transform_cmd, message):
+        proc = run(
+            transform_cmd,
+            input=message,
+            stdout=PIPE,
+            stderr=PIPE,
+            shell=True
+        )
+        if proc.returncode == 0:
+            return proc.stdout
+        raise ValueError(f'Transforming message {message} failed!')
+
+    def handle_event(self, message):
+        json_bytes = dumps(message).encode()
+        transformed_bytes = self._transform_out(json_bytes)
+        if transformed_bytes:
+            self.pipe_io.send_message(transformed_bytes)
+
+    def handle_pipe_event(self, message_bytes):
+        transformed_bytes = self._transform_in(message_bytes)
+        if transformed_bytes:
+            json_message = loads(transformed_bytes)
+            self.server_connector.send(json_message)
+
+
+class CommandEventHandler(PipeIOEventHandler):
+    def __init__(self, key, command, permissions=DEFAULT_PERMISSIONS):
+        super().__init__(
+            key,
+            self._generate_tempfilename(),
+            self._generate_tempfilename(),
+            permissions
+        )
+
+        self._proc_stdin = open(self.pipe_io.out_pipe, 'rb')
+        self._proc_stdout = open(self.pipe_io.in_pipe, 'wb')
+        self._proc = Popen(
+            command, shell=True, executable='/bin/bash',
+            stdin=self._proc_stdin, stdout=self._proc_stdout, stderr=PIPE,
+            start_new_session=True
+        )
+        self._monitor_proc_thread = self._start_monitor_proc()
+
+    def _generate_tempfilename(self):
+        # pylint: disable=no-self-use
+        random_filename = partial(token_urlsafe, 10)
+        return join('/tmp', f'{type(self).__name__}.{random_filename()}')
+
+    def _start_monitor_proc(self):
+        thread = Thread(target=self._monitor_proc, daemon=True)
+        thread.start()
+        return thread
+
+    @terminate_process_on_failure
+    def _monitor_proc(self):
+        return_code = self._proc.wait()
+        if return_code == -int(SIGTERM):
+            # supervisord asked the program to terminate, this is fine
+            return
+        if return_code != 0:
+            _, stderr = self._proc.communicate()
+            raise RuntimeError(f'Subprocess failed ({return_code})! Stderr:\n{stderr.decode()}')
+
+    def cleanup(self):
+        with suppress(ProcessLookupError):
+            process_group_id = getpgid(self._proc.pid)
+            killpg(process_group_id, SIGTERM)
+            self._proc_stdin.close()
+            self._proc_stdout.close()
+        super().cleanup()

lib/tfw/components/pipe_io_server/__init__.py

@@ -0,0 +1,2 @@
+from .pipe_io_server import PipeIOServer
+from .terminate_process_on_failure import terminate_process_on_failure

lib/tfw/components/pipe_io_server/deque.py

@@ -0,0 +1,27 @@
+from collections import deque
+from threading import Lock, Condition
+
+
+class Deque:
+    def __init__(self):
+        self._queue = deque()
+
+        self._mutex = Lock()
+        self._not_empty = Condition(self._mutex)
+
+    def pop(self):
+        with self._mutex:
+            while not self._queue:
+                self._not_empty.wait()
+            return self._queue.pop()
+
+    def push(self, item):
+        self._push(item, self._queue.appendleft)
+
+    def push_front(self, item):
+        self._push(item, self._queue.append)
+
+    def _push(self, item, put_method):
+        with self._mutex:
+            put_method(item)
+            self._not_empty.notify()

lib/tfw/components/pipe_io_server/pipe.py

@@ -0,0 +1,16 @@
+from os import mkfifo, remove, chmod
+from os.path import exists
+
+
+class Pipe:
+    def __init__(self, path):
+        self.path = path
+
+    def recreate(self, permissions):
+        self.remove()
+        mkfifo(self.path)
+        chmod(self.path, permissions)  # use chmod to ignore umask
+
+    def remove(self):
+        if exists(self.path):
+            remove(self.path)

lib/tfw/components/pipe_io_server/pipe_io_server.py

@@ -0,0 +1,73 @@
+from abc import ABC, abstractmethod
+from threading import Thread, Event
+from typing import Callable
+
+from .pipe_reader_thread import PipeReaderThread
+from .pipe_writer_thread import PipeWriterThread
+from .pipe import Pipe
+from .terminate_process_on_failure import terminate_process_on_failure
+
+
+class PipeIOServer(ABC, Thread):
+    def __init__(self, in_pipe=None, out_pipe=None, permissions=0o600):
+        super().__init__(daemon=True)
+        self._in_pipe, self._out_pipe = in_pipe, out_pipe
+        self._create_pipes(permissions)
+        self._stop_event = Event()
+        self._reader_thread, self._writer_thread = self._create_io_threads()
+        self._io_threads = (self._reader_thread, self._writer_thread)
+        self._on_stop = lambda: None
+
+    def _create_pipes(self, permissions):
+        Pipe(self.in_pipe).recreate(permissions)
+        Pipe(self.out_pipe).recreate(permissions)
+
+    @property
+    def in_pipe(self):
+        return self._in_pipe
+
+    @property
+    def out_pipe(self):
+        return self._out_pipe
+
+    def _create_io_threads(self):
+        reader_thread = PipeReaderThread(self.in_pipe, self._stop_event, self.handle_message)
+        writer_thread = PipeWriterThread(self.out_pipe, self._stop_event)
+        return reader_thread, writer_thread
+
+    @abstractmethod
+    def handle_message(self, message):
+        raise NotImplementedError()
+
+    def send_message(self, message):
+        self._writer_thread.write(message)
+
+    @terminate_process_on_failure
+    def run(self):
+        for thread in self._io_threads:
+            thread.start()
+        self._stop_event.wait()
+        self._stop_threads()
+
+    def stop(self):
+        self._stop_event.set()
+        if self.is_alive():
+            self.join()
+
+    def _stop_threads(self):
+        for thread in self._io_threads:
+            if thread.is_alive():
+                thread.stop()
+        Pipe(self.in_pipe).remove()
+        Pipe(self.out_pipe).remove()
+        self._on_stop()
+
+    def _set_on_stop(self, value):
+        if not isinstance(value, Callable):
+            raise ValueError("Supplied object is not callable!")
+        self._on_stop = value
+
+    on_stop = property(fset=_set_on_stop)
+
+    def wait(self):
+        self._stop_event.wait()

lib/tfw/components/pipe_io_server/pipe_reader_thread.py

@@ -0,0 +1,44 @@
+from contextlib import suppress
+from os import open as osopen
+from os import write, close, O_WRONLY, O_NONBLOCK
+from threading import Thread
+
+from .terminate_process_on_failure import terminate_process_on_failure
+
+
+class PipeReaderThread(Thread):
+    eof = b''
+    stop_sequence = b'stop_reading\n'
+
+    def __init__(self, pipe_path, stop_event, message_handler):
+        super().__init__(daemon=True)
+        self._message_handler = message_handler
+        self._pipe_path = pipe_path
+        self._stop_event = stop_event
+
+    @terminate_process_on_failure
+    def run(self):
+        with self._open() as pipe:
+            while True:
+                message = pipe.readline()
+                if message == self.stop_sequence:
+                    self._stop_event.set()
+                    break
+                if message == self.eof:
+                    self._open().close()
+                    continue
+                self._message_handler(message[:-1])
+
+    def _open(self):
+        return open(self._pipe_path, 'rb')
+
+    def stop(self):
+        while self.is_alive():
+            self._unblock()
+        self.join()
+
+    def _unblock(self):
+        with suppress(OSError):
+            fd = osopen(self._pipe_path, O_WRONLY | O_NONBLOCK)
+            write(fd, self.stop_sequence)
+            close(fd)

lib/tfw/components/pipe_io_server/pipe_writer_thread.py

@@ -0,0 +1,50 @@
+from contextlib import suppress
+from os import O_NONBLOCK, O_RDONLY, close
+from os import open as osopen
+from threading import Thread
+
+from .terminate_process_on_failure import terminate_process_on_failure
+from .deque import Deque
+
+
+class PipeWriterThread(Thread):
+    def __init__(self, pipe_path, stop_event):
+        super().__init__(daemon=True)
+        self._pipe_path = pipe_path
+        self._stop_event = stop_event
+        self._write_queue = Deque()
+
+    def write(self, message):
+        self._write_queue.push(message)
+
+    @terminate_process_on_failure
+    def run(self):
+        with self._open() as pipe:
+            while True:
+                message = self._write_queue.pop()
+                if message is None:
+                    self._stop_event.set()
+                    break
+                try:
+                    pipe.write(message + b'\n')
+                    pipe.flush()
+                except BrokenPipeError:
+                    try:  # pipe was reopened, close() flushed the message
+                        pipe.close()
+                    except BrokenPipeError:  # close() discarded the message
+                        self._write_queue.push_front(message)
+                    pipe = self._open()
+
+    def _open(self):
+        return open(self._pipe_path, 'wb')
+
+    def stop(self):
+        while self.is_alive():
+            self._unblock()
+        self.join()
+
+    def _unblock(self):
+        with suppress(OSError):
+            fd = osopen(self._pipe_path, O_RDONLY | O_NONBLOCK)
+            self._write_queue.push_front(None)
+            close(fd)

lib/tfw/components/pipe_io_server/terminate_process_on_failure.py

@@ -0,0 +1,15 @@
+from functools import wraps
+from os import kill, getpid
+from signal import SIGTERM
+from traceback import print_exc
+
+
+def terminate_process_on_failure(fun):
+    @wraps(fun)
+    def wrapper(*args, **kwargs):
+        try:
+            return fun(*args, **kwargs)
+        except:  # pylint: disable=bare-except
+            print_exc()
+            kill(getpid(), SIGTERM)
+    return wrapper

lib/tfw/components/process_managing_event_handler.py

@@ -38,7 +38,6 @@ class ProcessManagingEventHandler(EventHandlerBase):
     """
     def __init__(self, key, dirmonitor=None, log_tail=0):
         super().__init__(key)
-        self.key = key
         self.monitor = dirmonitor
         self.processmanager = ProcessManager()
         self.log_tail = log_tail

lib/tfw/components/snapshot_provider.py

@@ -0,0 +1,175 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+import re
+from subprocess import run, CalledProcessError
+from getpass import getuser
+from os.path import isdir
+from datetime import datetime
+from uuid import uuid4
+
+
+class SnapshotProvider:
+    def __init__(self, directory, git_dir):
+        self._classname = self.__class__.__name__
+        author = f'{getuser()} via TFW {self._classname}'
+        self.gitenv = {
+            'GIT_DIR': git_dir,
+            'GIT_WORK_TREE': directory,
+            'GIT_AUTHOR_NAME': author,
+            'GIT_AUTHOR_EMAIL': '',
+            'GIT_COMMITTER_NAME': author,
+            'GIT_COMMITTER_EMAIL': '',
+            'GIT_PAGER': 'cat'
+        }
+
+        self._init_repo()
+        self.__last_valid_branch = self._branch
+
+    def _init_repo(self):
+        self._check_environment()
+
+        if not self._repo_is_initialized:
+            self._run(('git', 'init'))
+
+        if self._number_of_commits == 0:
+            try:
+                self._snapshot()
+            except CalledProcessError:
+                raise EnvironmentError(f'{self._classname} cannot init on empty directories!')
+
+        self._check_head_not_detached()
+
+    def _check_environment(self):
+        if not isdir(self.gitenv['GIT_DIR']) or not isdir(self.gitenv['GIT_WORK_TREE']):
+            raise EnvironmentError(f'{self._classname}: "directory" and "git_dir" must exist!')
+
+    @property
+    def _repo_is_initialized(self):
+        return self._run(
+            ('git', 'status'),
+            check=False
+        ).returncode == 0
+
+    @property
+    def _number_of_commits(self):
+        return int(
+            self._get_stdout((
+                'git', 'rev-list',
+                '--all',
+                '--count'
+            ))
+        )
+
+    def _snapshot(self):
+        self._run((
+            'git', 'add',
+            '-A'
+        ))
+        self._run((
+            'git', 'commit',
+            '-m', 'Snapshot'
+        ))
+
+    def _check_head_not_detached(self):
+        if self._head_detached:
+            raise EnvironmentError(f'{self._classname} cannot init from detached HEAD state!')
+
+    @property
+    def _head_detached(self):
+        return self._branch == 'HEAD'
+
+    @property
+    def _branch(self):
+        return self._get_stdout((
+            'git', 'rev-parse',
+            '--abbrev-ref', 'HEAD'
+        ))
+
+    def _get_stdout(self, *args, **kwargs):
+        kwargs['capture_output'] = True
+        stdout_bytes = self._run(*args, **kwargs).stdout
+        return stdout_bytes.decode().rstrip('\n')
+
+    def _run(self, *args, **kwargs):
+        if 'check' not in kwargs:
+            kwargs['check'] = True
+        if 'env' not in kwargs:
+            kwargs['env'] = self.gitenv
+        return run(*args, **kwargs)
+
+    def take_snapshot(self):
+        if self._head_detached:
+            self._checkout_new_branch_from_head()
+        self._snapshot()
+
+    def _checkout_new_branch_from_head(self):
+        branch_name = uuid4()
+        self._run((
+            'git', 'branch',
+            branch_name
+        ))
+        self._checkout(branch_name)
+
+    def _checkout(self, what):
+        self._run((
+            'git', 'checkout',
+            what
+        ))
+
+    def restore_snapshot(self, date):
+        commit = self._get_commit_from_timestamp(date)
+        self._checkout(commit)
+
+    def _get_commit_from_timestamp(self, date):
+        return self._get_stdout((
+            'git', 'rev-list',
+            '--date=iso',
+            '-n', '1',
+            f'--before="{date.isoformat()}"',
+            self._last_valid_branch
+        ))
+
+    @property
+    def _last_valid_branch(self):
+        if not self._head_detached:
+            self.__last_valid_branch = self._branch
+        return self.__last_valid_branch
+
+    @property
+    def all_timelines(self):
+        return self._branches
+
+    @property
+    def _branches(self):
+        git_branch_output = self._get_stdout(('git', 'branch'))
+        regex_pattern = re.compile(r'(?:[^\S\n]|[*])')  # matches '*' and non-newline whitespace chars
+        return re.sub(regex_pattern, '', git_branch_output).splitlines()
+
+    @property
+    def timeline(self):
+        return self._last_valid_branch
+
+    @timeline.setter
+    def timeline(self, value):
+        self._checkout(value)
+
+    @property
+    def snapshots(self):
+        return self._pretty_log_branch()
+
+    def _pretty_log_branch(self):
+        git_log_output = self._get_stdout((
+            'git', 'log',
+            '--pretty=%H@%aI'
+        ))
+
+        commits = []
+        for line in git_log_output.splitlines():
+            commit_hash, timestamp = line.split('@')
+            commits.append({
+                'hash': commit_hash,
+                'timestamp': datetime.fromisoformat(timestamp)
+            })
+
+        return commits

lib/tfw/components/terminal_commands.py

@@ -11,6 +11,7 @@ LOG = logging.getLogger(__name__)
 
 
 class TerminalCommands(ABC):
+    # pylint: disable=anomalous-backslash-in-string
     """
     A class you can use to define hooks for terminal commands. This means that you can
     have python code executed when the user enters a specific command to the terminal on

lib/tfw/components/terminal_event_handler.py

@@ -68,6 +68,7 @@ class TerminalEventHandler(EventHandlerBase):
                      (command to be written to the pty)
         """
         self.terminado_server.pty.write(data['value'])
+        return data
 
     def read(self, data):
         """

lib/tfw/crypto.py

@@ -0,0 +1,107 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from functools import wraps
+from base64 import b64encode, b64decode
+from copy import deepcopy
+from hashlib import md5
+from os import urandom, chmod
+from os.path import exists
+from stat import S_IRUSR, S_IWUSR, S_IXUSR
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.hashes import SHA256
+from cryptography.hazmat.primitives.hmac import HMAC as _HMAC
+from cryptography.exceptions import InvalidSignature
+
+from tfw.networking import message_bytes
+from tfw.decorators import lazy_property
+from tfw.config import TFWENV
+
+
+def message_checksum(message):
+    return md5(message_bytes(message)).hexdigest()
+
+
+def sign_message(key, message):
+    signature = message_signature(key, message)
+    message['signature'] = b64encode(signature).decode()
+
+
+def message_signature(key, message):
+    return HMAC(key, message_bytes(message)).signature
+
+
+def verify_message(key, message):
+    message = deepcopy(message)
+    try:
+        signature_b64 = message.pop('signature')
+        signature = b64decode(signature_b64)
+        actual_signature = message_signature(key, message)
+        return signature == actual_signature
+    except KeyError:
+        return False
+
+
+class KeyManager:
+    def __init__(self):
+        self.keyfile = TFWENV.AUTH_KEY
+        if not exists(self.keyfile):
+            self._init_auth_key()
+
+    @lazy_property
+    def auth_key(self):
+        with open(self.keyfile, 'rb') as ifile:
+            return ifile.read()
+
+    def _init_auth_key(self):
+        key = self.generate_key()
+        with open(self.keyfile, 'wb') as ofile:
+            ofile.write(key)
+        self._chmod_700_keyfile()
+        return key
+
+    @staticmethod
+    def generate_key():
+        return urandom(32)
+
+    def _chmod_700_keyfile(self):
+        chmod(self.keyfile, S_IRUSR | S_IWUSR | S_IXUSR)
+
+
+class HMAC:
+    def __init__(self, key, message):
+        self.key = key
+        self.message = message
+        self._hmac = _HMAC(
+            key=key,
+            algorithm=SHA256(),
+            backend=default_backend()
+        )
+
+    def _reload_if_finalized(f):
+        # pylint: disable=no-self-argument,not-callable
+        @wraps(f)
+        def wrapped(instance, *args, **kwargs):
+            if getattr(instance, '_finalized', False):
+                instance.__init__(instance.key, instance.message)
+            ret_val = f(instance, *args, **kwargs)
+            setattr(instance, '_finalized', True)
+            return ret_val
+        return wrapped
+
+    @property
+    @_reload_if_finalized
+    def signature(self):
+        self._hmac.update(self.message)
+        signature = self._hmac.finalize()
+        return signature
+
+    @_reload_if_finalized
+    def verify(self, signature):
+        self._hmac.update(self.message)
+        try:
+            self._hmac.verify(signature)
+            return True
+        except InvalidSignature:
+            return False

lib/tfw/decorators/lazy_property.py

@@ -1,6 +1,8 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
+from functools import update_wrapper
+
 
 class lazy_property:
     """
@@ -9,7 +11,7 @@ class lazy_property:
     """
     def __init__(self, func):
         self.func = func
-        self.__doc__ = func.__doc__
+        update_wrapper(self, func)
 
     def __get__(self, instance, owner):
         if instance is None:

lib/tfw/decorators/rate_limiter.py

@@ -1,25 +1,100 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from functools import wraps
+from functools import wraps, partial
 from time import time, sleep
 
+from tfw.decorators.lazy_property import lazy_property
+
 
 class RateLimiter:
+    """
+    Decorator class for rate limiting, blocking.
+
+    When applied to a function this decorator will apply rate limiting
+    if the function is invoked more frequently than rate_per_seconds.
+
+    By default rate limiting means sleeping until the next invocation time
+    as per __init__ parameter rate_per_seconds.
+
+    Note that this decorator BLOCKS THE THREAD it is being executed on,
+    so it is only acceptable for stuff running on a separate thread.
+
+    If this is no good for you please refer to AsyncRateLimiter in this module,
+    which is designed not to block and use the IOLoop it is being called from.
+    """
     def __init__(self, rate_per_second):
+        """
+        :param rate_per_second: max frequency the decorated method should be
+                                invoked with
+        """
         self.min_interval = 1 / float(rate_per_second)
+        self.fun = None
         self.last_call = time()
 
+    def action(self, seconds_to_next_call):
+        if seconds_to_next_call:
+            sleep(seconds_to_next_call)
+        self.fun()
+
     def __call__(self, fun):
         @wraps(fun)
         def wrapper(*args, **kwargs):
-            self._limit_rate()
-            fun(*args, **kwargs)
+            self.fun = partial(fun, *args, **kwargs)
+            limit_seconds = self._limit_rate()
+            self.action(limit_seconds)
         return wrapper
 
     def _limit_rate(self):
-        since_last_call = time() - self.last_call
-        to_next_call = self.min_interval - since_last_call
+        seconds_since_last_call = time() - self.last_call
+        seconds_to_next_call = self.min_interval - seconds_since_last_call
+
+        if seconds_to_next_call > 0:
+            return seconds_to_next_call
         self.last_call = time()
-        if to_next_call > 0:
-            sleep(to_next_call)
+        return 0
+
+
+class AsyncRateLimiter(RateLimiter):
+    """
+    Decorator class for rate limiting, non-blocking.
+
+    The semantics of the rate limiting:
+        - unlike RateLimiter this decorator never blocks, instead it adds an async
+          callback version of the decorated function to the IOLoop
+          (to be executed after the rate limiting has expired).
+        - the timing works similarly to RateLimiter
+    """
+    def __init__(self, rate_per_second, ioloop_factory):
+        """
+        :param rate_per_second: max frequency the decorated method should be
+                                invoked with
+        :param ioloop_factory: callable that should return an instance of the
+                               IOLoop of the application
+        """
+        self._ioloop_factory = ioloop_factory
+        self._ioloop = None
+        self._last_callback = None
+
+        self._make_action_thread_safe()
+        super().__init__(rate_per_second=rate_per_second)
+
+    def _make_action_thread_safe(self):
+        self.action = partial(self.ioloop.add_callback, self.action)
+
+    @lazy_property
+    def ioloop(self):
+        return self._ioloop_factory()
+
+    def action(self, seconds_to_next_call):
+        if self._last_callback:
+            self.ioloop.remove_timeout(self._last_callback)
+
+        self._last_callback = self.ioloop.call_later(
+            seconds_to_next_call,
+            self.fun_with_debounce
+        )
+
+    def fun_with_debounce(self):
+        self.last_call = time()
+        self.fun()

lib/tfw/event_handler_base.py

@@ -2,9 +2,14 @@
 # All Rights Reserved. See LICENSE file for details.
 
 from abc import ABC, abstractmethod
+from inspect import currentframe
+from typing import Iterable
 
-from tfw.networking import deserialize_tfw_msg
 from tfw.networking.event_handlers import ServerConnector
+from tfw.crypto import message_checksum, KeyManager, verify_message
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
 
 
 class EventHandlerBase(ABC):
@@ -16,30 +21,58 @@ class EventHandlerBase(ABC):
     """
     def __init__(self, key):
         self.server_connector = ServerConnector()
-        self.key = key
-        self.subscribe(self.key, 'reset')
+        self.keys = []
+        if isinstance(key, str):
+            self.keys.append(key)
+        elif isinstance(key, Iterable):
+            self.keys = list(key)
+
+        self.subscribe(*self.keys)
         self.server_connector.register_callback(self.event_handler_callback)
 
-    def event_handler_callback(self, msg_parts):
+    @property
+    def key(self):
+        """
+        Returns the oldest key this EventHandler was subscribed to.
+        """
+        return self.keys[0]
+
+    def event_handler_callback(self, message):
         """
         Callback that is invoked when receiving a message.
         Dispatches messages to handler methods and sends
         a response back in case the handler returned something.
         This is subscribed in __init__().
         """
-        message = deserialize_tfw_msg(*msg_parts)
+        if not self.check_key(message):
+            return
+
         response = self.dispatch_handling(message)
         if response:
-            response['key'] = message['key']
             self.server_connector.send(response)
 
+    def check_key(self, message):
+        """
+        Checks whether the message is intended for this
+        EventHandler.
+
+        This is necessary because ZMQ handles PUB - SUB
+        connetions with pattern matching (e.g. someone
+        subscribed to 'fsm' will receive 'fsm_update'
+        messages as well.
+        """
+        if '' in self.keys:
+            return True
+        return message['key'] in self.keys
+
     def dispatch_handling(self, message):
         """
         Used to dispatch messages to their specific handlers.
+
+        :param message: the message received
+        :returns: the message to send back
         """
-        if message['key'] != 'reset':
         return self.handle_event(message)
-        return self.handle_reset(message)
 
     @abstractmethod
     def handle_event(self, message):
@@ -47,18 +80,10 @@ class EventHandlerBase(ABC):
         Abstract method that implements the handling of messages.
 
         :param message: the message received
+        :returns: the message to send back
         """
         raise NotImplementedError
 
-    def handle_reset(self, message):
-        # pylint: disable=unused-argument,no-self-use
-        """
-        Usually 'reset' events receive some sort of special treatment.
-
-        :param message: the message received
-        """
-        return None
-
     def subscribe(self, *keys):
         """
         Subscribe this EventHandler to receive events for given keys.
@@ -70,6 +95,7 @@ class EventHandlerBase(ABC):
         """
         for key in keys:
             self.server_connector.subscribe(key)
+            self.keys.append(key)
 
     def unsubscribe(self, *keys):
         """
@@ -79,6 +105,7 @@ class EventHandlerBase(ABC):
         """
         for key in keys:
             self.server_connector.unsubscribe(key)
+            self.keys.remove(key)
 
     def cleanup(self):
         """
@@ -87,18 +114,75 @@ class EventHandlerBase(ABC):
         """
         pass
 
+    @classmethod
+    def get_local_instances(cls):
+        frame = currentframe()
+        if frame is None:
+            raise EnvironmentError('inspect.currentframe() is not supported!')
 
-class TriggeredEventHandler(EventHandlerBase, ABC):
+        locals_values = frame.f_back.f_locals.values()
+        return {
+            instance for instance in locals_values
+            if isinstance(instance, cls)
+        }
+
+
+class FSMAwareEventHandler(EventHandlerBase, ABC):
     # pylint: disable=abstract-method
     """
-    Abstract base class for EventHandlers which are only triggered in case
-    TFWServer has successfully triggered an FSM step defined in __init__.
+    Abstract base class for EventHandlers which automatically
+    keep track of the state of the TFW FSM.
     """
-    def __init__(self, key, trigger):
+    def __init__(self, key):
         super().__init__(key)
-        self.trigger = trigger
+        self.subscribe('fsm_update')
+        self.fsm_state = None
+        self.in_accepted_state = False
+        self._auth_key = KeyManager().auth_key
 
     def dispatch_handling(self, message):
-        if message.get('trigger') == self.trigger:
-            return super().dispatch_handling(message)
+        if message['key'] == 'fsm_update':
+            if verify_message(self._auth_key, message):
+                self._handle_fsm_update(message)
             return None
+        return super().dispatch_handling(message)
+
+    def _handle_fsm_update(self, message):
+        try:
+            new_state = message['data']['current_state']
+            trigger = message['data']['last_trigger']
+            if self.fsm_state != new_state:
+                self.handle_fsm_step(self.fsm_state, new_state, trigger)
+            self.fsm_state = new_state
+            self.in_accepted_state = message['data']['in_accepted_state']
+        except KeyError:
+            LOG.error('Invalid fsm_update message received!')
+
+    def handle_fsm_step(self, from_state, to_state, trigger):
+        """
+        Called in case the TFW FSM has stepped.
+        """
+        pass
+
+
+class BroadcastingEventHandler(EventHandlerBase, ABC):
+    # pylint: disable=abstract-method
+    """
+    Abstract base class for EventHandlers which broadcast responses
+    and intelligently ignore their own broadcasted messages they receive.
+    """
+    def __init__(self, key):
+        super().__init__(key)
+        self.own_message_hashes = []
+
+    def event_handler_callback(self, message):
+        message_hash = message_checksum(message)
+
+        if message_hash in self.own_message_hashes:
+            self.own_message_hashes.remove(message_hash)
+            return
+
+        response = self.dispatch_handling(message)
+        if response:
+            self.own_message_hashes.append(message_checksum(response))
+            self.server_connector.broadcast(response)

lib/tfw/fsm_base.py

@@ -1,27 +1,33 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from typing import List
+from collections import defaultdict
 
-from transitions import Machine
+from transitions import Machine, MachineError
 
 from tfw.mixins import CallbackMixin
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
 
 
-class FSMBase(CallbackMixin):
+class FSMBase(Machine, CallbackMixin):
     """
     A general FSM base class you can inherit from to track user progress.
     See linear_fsm.py for an example use-case.
-    TFW the transitions library for state machines, please refer to their
+    TFW uses the transitions library for state machines, please refer to their
     documentation for more information on creating your own machines:
     https://github.com/pytransitions/transitions
     """
     states, transitions = [], []
 
-    def __init__(self, initial: str = None, accepted_states: List[str] = None):
-        self.accepted_states = accepted_states or [self.states[-1]]
-        self.machine = Machine(
-            model=self,
+    def __init__(self, initial=None, accepted_states=None):
+        self.accepted_states = accepted_states or [self.states[-1].name]
+        self.trigger_predicates = defaultdict(list)
+        self.trigger_history = []
+
+        Machine.__init__(
+            self,
             states=self.states,
             transitions=self.transitions,
             initial=initial or self.states[0],
@@ -35,3 +41,28 @@ class FSMBase(CallbackMixin):
 
     def is_solved(self):
         return self.state in self.accepted_states  # pylint: disable=no-member
+
+    def subscribe_predicate(self, trigger, *predicates):
+        self.trigger_predicates[trigger].extend(predicates)
+
+    def unsubscribe_predicate(self, trigger, *predicates):
+        self.trigger_predicates[trigger] = [
+            predicate
+            for predicate in self.trigger_predicates[trigger]
+            not in predicates
+        ]
+
+    def step(self, trigger):
+        predicate_results = (
+            predicate()
+            for predicate in self.trigger_predicates[trigger]
+        )
+
+        if all(predicate_results):
+            try:
+                self.trigger(trigger)
+                self.trigger_history.append(trigger)
+                return True
+            except (AttributeError, MachineError):
+                LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
+        return False

lib/tfw/linear_fsm.py

@@ -1,6 +1,8 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
+from transitions import State
+
 from .fsm_base import FSMBase
 
 
@@ -13,17 +15,17 @@ class LinearFSM(FSMBase):
     (0) --  step_1  --> (1) --  step_2  --> (2) --  step_3  --> (3) ... and so on
     """
     def __init__(self, number_of_steps):
-        self.states = list(map(str, range(number_of_steps)))
+        self.states = [State(name=str(index)) for index in range(number_of_steps)]
         self.transitions = []
-        for index in self.states[:-1]:
+        for state in self.states[:-1]:
             self.transitions.append({
-                'trigger': f'step_{int(index)+1}',
-                'source': index,
-                'dest': str(int(index)+1)
+                'trigger': f'step_{int(state.name)+1}',
+                'source': state.name,
+                'dest': str(int(state.name)+1)
             })
             self.transitions.append({
                 'trigger': 'step_next',
-                'source': index,
-                'dest': str(int(index)+1)
+                'source': state.name,
+                'dest': str(int(state.name)+1)
             })
         super(LinearFSM, self).__init__()

lib/tfw/mixins/callback_mixin.py

@@ -9,6 +9,7 @@ from tfw.decorators import lazy_property
 class CallbackMixin:
     @lazy_property
     def _callbacks(self):
+        # pylint: disable=no-self-use
         return []
 
     def subscribe_callback(self, callback, *args, **kwargs):
@@ -21,6 +22,14 @@ class CallbackMixin:
         fun = partial(callback, *args, **kwargs)
         self._callbacks.append(fun)
 
+    def subscribe_callbacks(self, *callbacks):
+        """
+        Subscribe a list of callbacks to incoke once an event is triggered.
+        :param callbacks: callbacks to be subscribed
+        """
+        for callback in callbacks:
+            self.subscribe_callback(callback)
+
     def unsubscribe_callback(self, callback):
         self._callbacks.remove(callback)
 

lib/tfw/mixins/observer_mixin.py

@@ -9,6 +9,7 @@ from tfw.decorators import lazy_property
 class ObserverMixin:
     @lazy_property
     def observer(self):
+        # pylint: disable=no-self-use
         return Observer()
 
     def watch(self):

lib/tfw/mixins/supervisor_mixin.py

@@ -13,6 +13,7 @@ from tfw.config import TFWENV
 class SupervisorBaseMixin:
     @lazy_property
     def supervisor(self):
+        # pylint: disable=no-self-use
         return xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor
 
 

lib/tfw/networking/__init__.py

@@ -1,9 +1,9 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from .serialization import serialize_tfw_msg, deserialize_tfw_msg, validate_message
+from .serialization import serialize_tfw_msg, deserialize_tfw_msg
+from .serialization import with_deserialize_tfw_msg, message_bytes
 from .zmq_connector_base import ZMQConnectorBase
-# from .controller_connector import ControllerConnector # TODO: readd once controller stuff is resolved
 from .message_sender import MessageSender
 from .event_handlers.server_connector import ServerUplinkConnector as TFWServerConnector
 from .server.tfw_server import TFWServer

lib/tfw/networking/controller_connector.py

@@ -1,18 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-import zmq
-from zmq.eventloop.zmqstream import ZMQStream
-
-from tfw.config import TFWENV
-from tfw.networking import ZMQConnectorBase
-
-
-class ControllerConnector(ZMQConnectorBase):
-    def __init__(self, zmq_context=None):
-        super(ControllerConnector, self).__init__(zmq_context)
-        self._zmq_rep_socket = self._zmq_context.socket(zmq.REP)
-        self._zmq_rep_socket.connect(f'tcp://localhost:{TFWENV.CONTROLLER_PORT}')
-        self._zmq_rep_stream = ZMQStream(self._zmq_rep_socket)
-
-        self.register_callback = self._zmq_rep_stream.on_recv_stream

lib/tfw/networking/event_handlers/server_connector.py

@@ -6,9 +6,12 @@ from functools import partial
 import zmq
 from zmq.eventloop.zmqstream import ZMQStream
 
-from tfw.networking import serialize_tfw_msg
+from tfw.networking import serialize_tfw_msg, with_deserialize_tfw_msg
 from tfw.networking import ZMQConnectorBase
 from tfw.config import TFWENV
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
 
 
 class ServerDownlinkConnector(ZMQConnectorBase):
@@ -20,7 +23,10 @@ class ServerDownlinkConnector(ZMQConnectorBase):
 
         self.subscribe = partial(self._zmq_sub_socket.setsockopt_string, zmq.SUBSCRIBE)
         self.unsubscribe = partial(self._zmq_sub_socket.setsockopt_string, zmq.UNSUBSCRIBE)
-        self.register_callback = self._zmq_sub_stream.on_recv
+
+    def register_callback(self, callback):
+        callback = with_deserialize_tfw_msg(callback)
+        self._zmq_sub_stream.on_recv(callback)
 
 
 class ServerUplinkConnector(ZMQConnectorBase):
@@ -34,26 +40,40 @@ class ServerUplinkConnector(ZMQConnectorBase):
 
     def send_to_eventhandler(self, message):
         """
-        Send a message to an event handler.
+        Send a message to an event handler through the TFW server.
+
         This envelopes the desired message in the 'data' field of the message to
         TFWServer, which will mirror it to event handlers.
 
         :param message: JSON message you want to send
-        :param message['key']: key of event handler you want to address
         """
-        nested_message = {'key': message['key'], 'data': message.pop('data')}
-        message['key'] = 'mirror'
-        message['data'] = nested_message
-        self.send(message)
+        self.send({
+            'key': 'mirror',
+            'data': message
+        })
 
     def send(self, message):
         """
-        Send a message to the TFW server
+        Send a message to the frontend through the TFW server.
 
         :param message: JSON message you want to send
         """
         self._zmq_push_socket.send_multipart(serialize_tfw_msg(message))
 
+    def broadcast(self, message):
+        """
+        Broadast a message through the TFW server.
+
+        This envelopes the desired message in the 'data' field of the message to
+        TFWServer, which will broadast it.
+
+        :param message: JSON message you want to send
+        """
+        self.send({
+            'key': 'broadcast',
+            'data': message
+        })
+
 
 class ServerConnector(ServerUplinkConnector, ServerDownlinkConnector):
     pass

lib/tfw/networking/message_sender.py

@@ -1,32 +1,54 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from datetime import datetime
-
 from tfw.networking.event_handlers import ServerUplinkConnector
 
 
 class MessageSender:
     """
-    Provides a mechanism to send messages to our frontend messaging component which
-    displays messages with the key "message".
+    Provides mechanisms to send messages to our frontend messaging component.
     """
-    def __init__(self, custom_key: str = None):
+    def __init__(self):
         self.server_connector = ServerUplinkConnector()
-        self.key = custom_key or 'message'
+        self.key = 'message'
+        self.queue_key = 'queueMessages'
 
     def send(self, originator, message):
         """
-        Sends a message to the key specified in __init__.
+        Sends a message.
         :param originator: name of sender to be displayed on the frontend
         :param message: message to send
         """
         data = {
             'originator': originator,
-            'timestamp': datetime.now().isoformat(),
             'message': message
         }
         self.server_connector.send({
             'key': self.key,
             'data': data
         })
+
+    def queue_messages(self, originator, messages):
+        """
+        Queues a list of messages to be displayed in a chatbot-like manner.
+        :param originator: name of sender to be displayed on the frontend
+        :param messages: list of messages to queue
+        """
+        data = {
+            'messages': [
+                {'message': message, 'originator': originator}
+                for message in messages
+            ]
+        }
+        self.server_connector.send({
+            'key': self.queue_key,
+            'data': data
+        })
+
+    @staticmethod
+    def generate_messages_from_queue(queue_message):
+        for message in queue_message['data']['messages']:
+            yield {
+                'key': 'message',
+                'data': message
+            }

lib/tfw/networking/serialization.py

@@ -22,10 +22,7 @@ The purpose of this module is abstracting away this low level behaviour.
 """
 
 import json
-
-
-def validate_message(message):
-    return 'key' in message
+from functools import wraps
 
 
 def serialize_tfw_msg(message):
@@ -35,6 +32,14 @@ def serialize_tfw_msg(message):
     return _serialize_all(message['key'], message)
 
 
+def with_deserialize_tfw_msg(fun):
+    @wraps(fun)
+    def wrapper(message_parts):
+        message = deserialize_tfw_msg(*message_parts)
+        return fun(message)
+    return wrapper
+
+
 def deserialize_tfw_msg(*args):
     """
     Return message from TFW multipart data
@@ -62,10 +67,14 @@ def _serialize_single(data):
     (serialize input if it is JSON)
     """
     if not isinstance(data, str):
-        data = json.dumps(data)
+        data = message_bytes(data)
     return _encode_if_needed(data)
 
 
+def message_bytes(message):
+    return json.dumps(message, sort_keys=True).encode()
+
+
 def _deserialize_single(data):
     """
     Try parsing input as JSON, return it as

lib/tfw/networking/server/__init__.py

@@ -3,5 +3,3 @@
 
 from .event_handler_connector import EventHandlerConnector, EventHandlerUplinkConnector, EventHandlerDownlinkConnector
 from .tfw_server import TFWServer
-from .zmq_websocket_handler import ZMQWebSocketProxy
-# from .controller_responder import ControllerResponder  # TODO: readd once controller stuff is resolved

lib/tfw/networking/server/controller_responder.py

@@ -1,38 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-from tfw.networking import deserialize_all, serialize_all, ControllerConnector
-
-
-class ControllerResponder:
-    def __init__(self, fsm):
-        self.fsm = fsm
-        self.token = None
-        self.controller_connector = ControllerConnector()
-        self.controller_connector.register_callback(self.handle_controller_request)
-        self.controller_request_handlers = {
-            'solution_check': self.handle_solution_check_request,
-            'test': self.handle_test_request,
-            'token': self.handle_token_request
-        }
-
-    def handle_controller_request(self, stream, msg_parts):
-        key, data = deserialize_all(*msg_parts)
-        response = self.controller_request_handlers[key](data)
-        stream.send_multipart(serialize_all(self.token, response))
-
-    def handle_test_request(self, data):
-        # pylint: disable=unused-argument,no-self-use
-        return 'OK'
-
-    def handle_token_request(self, data):
-        if self.token is None:
-            self.token = data
-        return {'token': self.token}
-
-    def handle_solution_check_request(self, data):
-        # pylint: disable=unused-argument
-        return {
-            'solved': self.fsm.is_solved(),
-            'message': 'solved' if self.fsm.is_solved() else 'not solved'
-        }

lib/tfw/networking/server/event_handler_connector.py

@@ -4,7 +4,7 @@
 import zmq
 from zmq.eventloop.zmqstream import ZMQStream
 
-from tfw.networking import ZMQConnectorBase, serialize_tfw_msg
+from tfw.networking import ZMQConnectorBase, serialize_tfw_msg, with_deserialize_tfw_msg
 from tfw.config import TFWENV
 from tfw.config.logs import logging
 
@@ -32,6 +32,7 @@ class EventHandlerUplinkConnector(ZMQConnectorBase):
 
 class EventHandlerConnector(EventHandlerDownlinkConnector, EventHandlerUplinkConnector):
     def register_callback(self, callback):
+        callback = with_deserialize_tfw_msg(callback)
         self._zmq_pull_stream.on_recv(callback)
 
     def send_message(self, message: dict):

lib/tfw/networking/server/tfw_server.py

@@ -1,15 +1,17 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from collections import defaultdict
+from abc import ABC, abstractmethod
+from contextlib import suppress
 
 from tornado.web import Application
 
-from tfw.networking import MessageSender
 from tfw.networking.event_handlers import ServerUplinkConnector
 from tfw.networking.server import EventHandlerConnector
+from tfw.networking import MessageSender
+from tfw.crypto import KeyManager, verify_message, sign_message
 from tfw.config.logs import logging
-from .zmq_websocket_handler import ZMQWebSocketProxy
+from .zmq_websocket_proxy import ZMQWebSocketProxy
 
 LOG = logging.getLogger(__name__)
 
@@ -18,117 +20,83 @@ class TFWServer:
     """
     This class handles the proxying of messages between the frontend and event handers.
     It proxies messages from the "/ws" route to all event handlers subscribed to a ZMQ
-    SUB socket. It also manages an FSM you can define as a constructor argument.
+    SUB socket.
     """
-    def __init__(self, fsm_type):
-        """
-        :param fsm_type: the type of FSM you want TFW to use
-        """
-        self._fsm = fsm_type()
-        self._fsm_updater = FSMUpdater(self._fsm)
-        self._fsm_manager = FSMManager(self._fsm)
-        self._fsm.subscribe_callback(self._fsm_updater.update)
+    def __init__(self):
         self._event_handler_connector = EventHandlerConnector()
+        self._uplink_connector = ServerUplinkConnector()
+        self._auth_key = KeyManager().auth_key
 
         self.application = Application([(
             r'/ws', ZMQWebSocketProxy, {
-                'make_eventhandler_message': self.make_eventhandler_message,
-                'proxy_filter': self.proxy_filter,
-                'handle_trigger': self.handle_trigger,
-                'event_handler_connector': self._event_handler_connector
-            })]
-        )
-        # self.controller_responder = ControllerResponder(self.fsm)
-        # TODO: add this once controller stuff is resolved
+                'event_handler_connector': self._event_handler_connector,
+                'message_handlers': [self.handle_trigger, self.handle_recover],
+                'frontend_message_handlers': [self.save_frontend_messages]
+            })])
 
-    @property
-    def fsm(self):
-        return self._fsm
-
-    @property
-    def fsm_manager(self):
-        return self._fsm_manager
-
-    def make_eventhandler_message(self, message):
-        self.trigger_fsm(message)
-        message['FSMUpdate'] = self._fsm_updater.get_fsm_state_and_transitions()
-        return message
+        self._frontend_messages = FrontendMessageStorage()
 
     def handle_trigger(self, message):
+        if 'trigger' in message:
             LOG.debug('Executing handler for trigger "%s"', message.get('trigger', ''))
-        self.trigger_fsm(message)
+            fsm_eh_command = {
+                'key': 'fsm',
+                'data': {
+                    'command': 'trigger',
+                    'value': message['trigger']
+                }
+            }
+            if verify_message(self._auth_key, message):
+                sign_message(self._auth_key, fsm_eh_command)
+            self._uplink_connector.send_to_eventhandler(fsm_eh_command)
 
-    def trigger_fsm(self, message):
-        trigger = message.get('trigger', '')
-        try:
-            self._fsm_manager.trigger(trigger, message)
-        except AttributeError:
-            LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
+    def handle_recover(self, message):
+        if message['key'] == 'recover':
+            self._frontend_messages.replay_messages(self._uplink_connector)
+            self._frontend_messages.clear()
 
-    def proxy_filter(self, message):
-        # pylint: disable=unused-argument,no-self-use
-        return True
+    def save_frontend_messages(self, message):
+        self._frontend_messages.save_message(message)
 
     def listen(self, port):
         self.application.listen(port)
 
 
-class FSMManager:
-    def __init__(self, fsm):
-        self._fsm = fsm
-        self.trigger_predicates = defaultdict(list)
-        self.messenge_sender = MessageSender()
+class MessageStorage(ABC):
+    def __init__(self):
+        self.saved_messages = []
 
-    @property
-    def fsm(self):
-        return self._fsm
+    def save_message(self, message):
+        with suppress(KeyError, AttributeError):
+            if self.filter_message(message):
+                self.saved_messages.extend(self.transform_message(message))
 
-    def trigger(self, trigger, message):
-        predicate_results = []
-        for predicate in self.trigger_predicates[trigger]:
-            success, message = predicate(message)
-            predicate_results.append(success)
-            self.messenge_sender.send('FSM', message)
+    @abstractmethod
+    def filter_message(self, message):
+        raise NotImplementedError
 
-        if all(predicate_results):
-            try:
-                self.fsm.trigger(trigger, message=message)
-            except AttributeError:
-                LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
+    def transform_message(self, message):   # pylint: disable=no-self-use
+        yield message
 
-    def subscribe_predicate(self, trigger, *predicates):
-        self.trigger_predicates[trigger].extend(predicates)
-
-    def unsubscribe_predicate(self, trigger, *predicates):
-        self.trigger_predicates[trigger] = [
-            predicate
-            for predicate in self.trigger_predicates[trigger]
-            not in predicates
-        ]
+    def clear(self):
+        self.saved_messages.clear()
 
 
-class FSMUpdater:
-    def __init__(self, fsm):
-        self.fsm = fsm
-        self.uplink = ServerUplinkConnector()
+class FrontendMessageStorage(MessageStorage):
+    def filter_message(self, message):
+        key = message['key']
+        command = message.get('data', {}).get('command')
+        return (
+            key in ('message', 'dashboard', 'queueMessages')
+            or key == 'ide' and command in ('select', 'read')
+        )
 
-    def update(self, kwargs_dict):
-        # pylint: disable=unused-argument
-        self.uplink.send(self.generate_fsm_update())
+    def transform_message(self, message):
+        if message['key'] == 'queueMessages':
+            yield from MessageSender.generate_messages_from_queue(message)
+        else:
+            yield message
 
-    def generate_fsm_update(self):
-        return {
-            'key': 'FSMUpdate',
-            'data': self.get_fsm_state_and_transitions()
-        }
-
-    def get_fsm_state_and_transitions(self):
-        state = self.fsm.state
-        valid_transitions = [
-            {'trigger': trigger}
-            for trigger in self.fsm.machine.get_triggers(self.fsm.state)
-        ]
-        return {
-            'current_state': state,
-            'valid_transitions': valid_transitions
-        }
+    def replay_messages(self, connector):
+        for message in self.saved_messages:
+            connector.send(message)

lib/tfw/networking/server/zmq_websocket_handler.py

@@ -1,91 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-import json
-from abc import ABC, abstractmethod
-
-from tornado.websocket import WebSocketHandler
-
-from tfw.networking import deserialize_tfw_msg, validate_message
-from tfw.config.logs import logging
-
-LOG = logging.getLogger(__name__)
-
-
-class ZMQWebSocketHandler(WebSocketHandler, ABC):
-    instances = set()
-
-    def initialize(self, **kwargs): # pylint: disable=arguments-differ
-        self._event_handler_connector = kwargs['event_handler_connector']
-
-    def prepare(self):
-        ZMQWebSocketHandler.instances.add(self)
-
-    def on_close(self):
-        ZMQWebSocketHandler.instances.remove(self)
-
-    def open(self, *args, **kwargs):
-        LOG.debug('WebSocket connection initiated')
-        self._event_handler_connector.register_callback(self.zmq_callback)
-
-    def zmq_callback(self, msg_parts):
-        keyhandlers = {'mirror': self.mirror}
-
-        message = deserialize_tfw_msg(*msg_parts)
-        LOG.debug('Received on pull socket: %s', message)
-        if not validate_message(message):
-            return
-
-        self.handle_trigger(message)
-        if message['key'] not in keyhandlers:
-            for instance in ZMQWebSocketHandler.instances:
-                instance.write_message(message)
-        else:
-            try:
-                keyhandlers[message['key']](message)
-            except KeyError:
-                LOG.error('Invalid mirror message format! Ignoring.')
-
-    def mirror(self, message):
-        message = message['data']
-        self._event_handler_connector.send_message(message)
-
-    def on_message(self, message):
-        LOG.debug('Received on WebSocket: %s', message)
-        if validate_message(message):
-            self.send_message(self.make_eventhandler_message(message))
-
-    @abstractmethod
-    def make_eventhandler_message(self, message):
-        raise NotImplementedError
-
-    def send_message(self, message: dict):
-        self._event_handler_connector.send_message(message)
-
-    @abstractmethod
-    def handle_trigger(self, message):
-        raise NotImplementedError
-
-    # much secure, very cors, wow
-    def check_origin(self, origin):
-        return True
-
-
-class ZMQWebSocketProxy(ZMQWebSocketHandler):
-    # pylint: disable=abstract-method
-    def initialize(self, **kwargs): # pylint: disable=arguments-differ
-        super(ZMQWebSocketProxy, self).initialize(**kwargs)
-        self._make_eventhandler_message = kwargs['make_eventhandler_message']
-        self._proxy_filter = kwargs['proxy_filter']
-        self._handle_trigger = kwargs['handle_trigger']
-
-    def on_message(self, message):
-        message = json.loads(message)
-        if self._proxy_filter(message):
-            super().on_message(message)
-
-    def make_eventhandler_message(self, message):
-        return self._make_eventhandler_message(message)
-
-    def handle_trigger(self, message):
-        self._handle_trigger(message)

lib/tfw/networking/server/zmq_websocket_proxy.py

@@ -0,0 +1,138 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+import json
+
+from tornado.websocket import WebSocketHandler
+
+from tfw.mixins import CallbackMixin
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class ZMQWebSocketProxy(WebSocketHandler):
+    # pylint: disable=abstract-method
+    instances = set()
+
+    def initialize(self, **kwargs): # pylint: disable=arguments-differ
+        self._event_handler_connector = kwargs['event_handler_connector']
+
+        self._message_handlers = kwargs.get('message_handlers', [])
+        self._frontend_message_handlers = kwargs.get('frontend_message_handlers', [])
+        self._eventhandler_message_handlers = kwargs.get('eventhandler_message_handlers', [])
+        self._proxy_filters = kwargs.get('proxy_filters', [])
+
+        self.proxy_eventhandler_to_websocket = TFWProxy(
+            self.send_eventhandler_message,
+            self.send_websocket_message
+        )
+        self.proxy_websocket_to_eventhandler = TFWProxy(
+            self.send_websocket_message,
+            self.send_eventhandler_message
+        )
+
+        self.subscribe_proxy_callbacks()
+
+    def subscribe_proxy_callbacks(self):
+        self.proxy_websocket_to_eventhandler.subscribe_proxy_callbacks_and_filters(
+            self._eventhandler_message_handlers + self._message_handlers,
+            self._proxy_filters
+        )
+
+        self.proxy_eventhandler_to_websocket.subscribe_proxy_callbacks_and_filters(
+            self._frontend_message_handlers + self._message_handlers,
+            self._proxy_filters
+        )
+
+    def prepare(self):
+        ZMQWebSocketProxy.instances.add(self)
+
+    def on_close(self):
+        ZMQWebSocketProxy.instances.remove(self)
+
+    def open(self, *args, **kwargs):
+        LOG.debug('WebSocket connection initiated')
+        self._event_handler_connector.register_callback(self.eventhander_callback)
+
+    def eventhander_callback(self, message):
+        """
+        Invoked on ZMQ messages from event handlers.
+        """
+        LOG.debug('Received on pull socket: %s', message)
+        self.proxy_eventhandler_to_websocket(message)
+
+    def on_message(self, message):
+        """
+        Invoked on WS messages from frontend.
+        """
+        message = json.loads(message)
+        LOG.debug('Received on WebSocket: %s', message)
+        self.proxy_websocket_to_eventhandler(message)
+
+    def send_eventhandler_message(self, message):
+        self._event_handler_connector.send_message(message)
+
+    @staticmethod
+    def send_websocket_message(message):
+        for instance in ZMQWebSocketProxy.instances:
+            instance.write_message(message)
+
+    # much secure, very cors, wow
+    def check_origin(self, origin):
+        return True
+
+
+class TFWProxy:
+    # pylint: disable=protected-access
+    def __init__(self, to_source, to_destination):
+        self.to_source = to_source
+        self.to_destination = to_destination
+
+        self.proxy_filters = CallbackMixin()
+        self.proxy_callbacks = CallbackMixin()
+
+        self.proxy_filters.subscribe_callback(self.validate_message)
+
+        self.keyhandlers = {
+            'mirror':    self.mirror,
+            'broadcast': self.broadcast
+        }
+
+    @staticmethod
+    def validate_message(message):
+        if 'key' not in message:
+            raise ValueError('Invalid TFW message format!')
+
+    def __call__(self, message):
+        try:
+            self.proxy_filters._execute_callbacks(message)
+        except ValueError:
+            LOG.exception('Invalid TFW message received!')
+            return
+
+        self.proxy_callbacks._execute_callbacks(message)
+
+        if message['key'] not in self.keyhandlers:
+            self.to_destination(message)
+        else:
+            handler = self.keyhandlers[message['key']]
+            try:
+                handler(message)
+            except KeyError:
+                LOG.error('Invalid "%s" message format! Ignoring.', handler.__name__)
+
+    def mirror(self, message):
+        message = message['data']
+        LOG.debug('Mirroring message: %s', message)
+        self.to_source(message)
+
+    def broadcast(self, message):
+        message = message['data']
+        LOG.debug('Broadcasting message: %s', message)
+        self.to_source(message)
+        self.to_destination(message)
+
+    def subscribe_proxy_callbacks_and_filters(self, proxy_callbacks, proxy_filters):
+        self.proxy_callbacks.subscribe_callbacks(*proxy_callbacks)
+        self.proxy_filters.subscribe_callbacks(*proxy_filters)

lib/tfw/yaml_fsm.py

@@ -0,0 +1,95 @@
+from subprocess import Popen, run
+from functools import partial, singledispatch
+from contextlib import suppress
+
+import yaml
+import jinja2
+from transitions import State
+
+from tfw import FSMBase
+
+
+class YamlFSM(FSMBase):
+    def __init__(self, config_file, jinja2_variables=None):
+        self.config = ConfigParser(config_file, jinja2_variables).config
+        self.setup_states()
+        super().__init__()  # FSMBase.__init__() requires states
+        self.setup_transitions()
+
+    def setup_states(self):
+        self.for_config_states_and_transitions_do(self.wrap_callbacks_with_subprocess_call)
+        self.states = [State(**state) for state in self.config['states']]
+
+    def setup_transitions(self):
+        self.for_config_states_and_transitions_do(self.subscribe_and_remove_predicates)
+        for transition in self.config['transitions']:
+            self.add_transition(**transition)
+
+    def for_config_states_and_transitions_do(self, what):
+        for array in ('states', 'transitions'):
+            for json_obj in self.config[array]:
+                what(json_obj)
+
+    @staticmethod
+    def wrap_callbacks_with_subprocess_call(json_obj):
+        topatch = ('on_enter', 'on_exit', 'prepare', 'before', 'after')
+        for key in json_obj:
+            if key in topatch:
+                json_obj[key] = partial(run_command_async, json_obj[key])
+
+    def subscribe_and_remove_predicates(self, json_obj):
+        if 'predicates' in json_obj:
+            for predicate in json_obj['predicates']:
+                self.subscribe_predicate(
+                    json_obj['trigger'],
+                    partial(
+                        command_statuscode_is_zero,
+                        predicate
+                        )
+                )
+
+        with suppress(KeyError):
+            json_obj.pop('predicates')
+
+
+def run_command_async(command, _):
+    Popen(command, shell=True)
+
+
+def command_statuscode_is_zero(command):
+    return run(command, shell=True).returncode == 0
+
+
+class ConfigParser:
+    def __init__(self, config_file, jinja2_variables):
+        self.read_variables = singledispatch(self._read_variables)
+        self.read_variables.register(dict, self._read_variables_dict)
+        self.read_variables.register(str, self._read_variables_str)
+
+        self.config = self.parse_config(config_file, jinja2_variables)
+
+    def parse_config(self, config_file, jinja2_variables):
+        config_string = self.read_file(config_file)
+        if jinja2_variables is not None:
+            variables = self.read_variables(jinja2_variables)
+            template = jinja2.Environment(loader=jinja2.BaseLoader).from_string(config_string)
+            config_string = template.render(**variables)
+        return yaml.safe_load(config_string)
+
+    @staticmethod
+    def read_file(filename):
+        with open(filename, 'r') as ifile:
+            return ifile.read()
+
+    @staticmethod
+    def _read_variables(variables):
+        raise TypeError(f'Invalid variables type {type(variables)}')
+
+    @staticmethod
+    def _read_variables_str(variables):
+        with open(variables, 'r') as ifile:
+            return yaml.safe_load(ifile)
+
+    @staticmethod
+    def _read_variables_dict(variables):
+        return variables

requirements.txt

@@ -3,3 +3,6 @@ pyzmq==17.0.0
 transitions==0.6.4
 terminado==0.8.1
 watchdog==0.8.3
+PyYAML==3.12
+Jinja2==2.10
+cryptography==2.2.2

setup.py

@@ -2,7 +2,6 @@ from os.path import dirname, realpath, join
 
 from setuptools import setup, find_packages
 
-
 here = dirname(realpath(__file__))
 
 with open(join(here, 'VERSION'), 'r') as ifile:
@@ -10,7 +9,8 @@ with open(join(here, 'VERSION'), 'r') as ifile:
 with open(join(here, 'requirements.txt'), 'r') as ifile:
     requirements = ifile.read().splitlines()
 
-setup(name='tfw',
+setup(
+    name = 'tfw',
     version = version,
     description = 'Avatao tutorial-framework',
     url = 'https://github.com/avatao-content/baseimage-tutorial-framework',
@@ -20,4 +20,10 @@ setup(name='tfw',
     packages = find_packages('lib'),
     package_dir = {'': 'lib'},
     install_requires = requirements,
-      zip_safe=False)
+    extras_require = {
+        'docs': [
+            'sphinx >= 1.7.0',
+        ],
+    },
+    zip_safe = False,
+)

supervisor/components/tfw_server.conf

@@ -0,0 +1,4 @@
+[program:tfwserver]
+user=root
+directory=%(ENV_TFW_SERVER_DIR)s
+command=python3 tfw_server.py

supervisor/supervisord.conf

@@ -4,6 +4,7 @@ logfile=/dev/null
 logfile_maxbytes=0
 loglevel = debug
 pidfile = /tmp/supervisord.pid
+strip_ansi=true
 
 [inet_http_server]
 port = 127.0.0.1:%(ENV_TFW_SUPERVISOR_HTTP_PORT)s

supervisor/tfw_server.py

@@ -0,0 +1,9 @@
+from tornado.ioloop import IOLoop
+
+from tfw.networking import TFWServer
+from tfw.config import TFWENV
+
+
+if __name__ == '__main__':
+    TFWServer().listen(TFWENV.WEB_PORT)
+    IOLoop.instance().start()