Add error handling logic to MessageStorage

Fix a Sphinx warning message

.drone.yml

@@ -10,4 +10,4 @@ pipeline:
       - docker push eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG}
     when:
       event: 'tag'
-      branch: refs/tags/egyptianmau-20*
+      branch: refs/tags/mainecoon-20*

.pylintrc

@@ -1,5 +1,5 @@
 [TYPECHECK]
 
 ignored-modules = zmq
-max-line-length = 150
-disable = missing-docstring, too-few-public-methods
+max-line-length = 120
+disable = missing-docstring, too-few-public-methods, invalid-name

Dockerfile

@@ -34,9 +34,11 @@ ENV PYTHONPATH="/usr/local/lib"                                            \
     TFW_NGINX_CONF="/etc/nginx/nginx.conf"                                 \
     TFW_NGINX_DEFAULT="/etc/nginx/sites-enabled/default"                   \
     TFW_NGINX_COMPONENTS="/etc/nginx/components"                           \
-    TFW_LIB_DIR="/usr/local/lib/"                                          \
+    TFW_LIB_DIR="/usr/local/lib"                                           \
     TFW_TERMINADO_DIR="/tmp/terminado_server"                              \
     TFW_FRONTEND_DIR="/srv/frontend"                                       \
+    TFW_SERVER_DIR="/srv/.tfw"                                             \
+    TFW_AUTH_KEY="/tmp/tfw-auth.key"                                       \
     TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history"                      \
     PROMPT_COMMAND="history -a"
 
@@ -45,13 +47,15 @@ RUN echo "export HISTFILE=${TFW_HISTFILE}" >> /tmp/bashrc &&\
     cat /tmp/bashrc >> /home/${AVATAO_USER}/.bashrc
 
 COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF}
+COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS}
 COPY nginx/nginx.conf ${TFW_NGINX_CONF}
 COPY nginx/default.conf ${TFW_NGINX_DEFAULT}
 COPY nginx/components/ ${TFW_NGINX_COMPONENTS}
-COPY lib LICENSE ${TFW_LIB_DIR}
+COPY lib LICENSE ${TFW_LIB_DIR}/
+COPY supervisor/tfw_server.py ${TFW_SERVER_DIR}/
 
-RUN for dir in "${TFW_LIB_DIR}"/{tfw,tao,envvars.py} "/etc/nginx" "/etc/supervisor"; do \
-        chown -R root:root "$dir" && chmod -R 700 "$dir";                               \
+RUN for dir in "${TFW_LIB_DIR}"/{tfw,tao,envvars} "/etc/nginx" "/etc/supervisor"; do \
+        chown -R root:root "$dir" && chmod -R 700 "$dir";                            \
     done
 
 ONBUILD ARG BUILD_CONTEXT="solvable"
@@ -63,7 +67,7 @@ ONBUILD COPY ${BUILD_CONTEXT}/supervisor/ ${TFW_SUPERVISORD_COMPONENTS}
 ONBUILD RUN for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do                               \
                 envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f~ && mv $f~ $f ;\
             done
-ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx"]
+ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx", "${TFW_LIB_DIR}/envvars", "${TFW_LIB_DIR}/tfw"]
 
 ONBUILD COPY ${BUILD_CONTEXT}/frontend /data/
 ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn install --frozen-lockfile || :

README.md

@@ -20,6 +20,20 @@ Frontend components use websockets to connect to the TFW server, to which you ca
 
 ![TFW architecture](docs/tfw_architecture.png)
 
+### Networking details
+
+Event handlers connect to the TFW server using ZMQ.
+They receive messages on their `SUB`(scribe) sockets, which are connected to the `PUB`(lish) socket of the server.
+Event handlers reply on their `PUSH` socket, then their messages are received on the `PULL` socket of the server.
+
+The TFW server is basically just a fancy proxy.
+It's behaviour is quite simple: it proxies every message received from the fontend to the event handlers and vice versa.
+
+The server is also capable of "mirroring" messages back to their source.
+This is useful for communication between event handlers or frontend components (event handler to event handler or frontend component to frontend component communication).
+
+Components can also broadcast messages (broadcasted messages are received both by event handlers and the frontend as well).
+
 ### Event handlers
 
 Imagine event handlers as callbacks that are invoked when TFW receives a specific type of message. For instance, you could send a message to the framework when the user does something of note.
@@ -75,9 +89,193 @@ The TFW message format:
 - The `data` object can contain anything you might want to send
 - The `trigger` key is an optional field that triggers an FSM action with that name from the current state (whatever that might be)
 
+To mirror messages back to their sources you can use a special messaging format, in which the message to be mirrored is enveloped inside the `data` field of the outer message:
+
+```text
+    "key": "mirror",
+    "data":
+    {
+        ...
+        The message you want to mirror (with it's own "key" and "data" fields)
+        ...
+    }
+```
+
+Broadcasting messages is possible in a similar manner by using `"key": "broadcast"` in the outer message.
 
 ## Where to go next
 
 Most of the components you need have docstrings included (hang on tight, this is work in progress) – refer to them for usage info.
 
-To get started you should take a look at the [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework) repository, which serves as an example project as well.
+In the `docs` folder you can find our Sphinx-based documentation, which you can build using the `hack/tfw.sh` script in the [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework) repository.
+
+To get started you should take a look at [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework), which serves as an example project as well.
+
+## API
+
+APIs exposed by our pre-witten event handlers are documented here.
+
+### IdeEventHandler
+
+You can read the content of the currently selected file like so:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "read"
+    }
+}
+```
+
+Use the following message to overwrite the content of the currently selected file:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "write",
+        "content": ...string...
+    }
+}
+```
+
+To select a file use the following message:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "select",
+        "filename": ...string...
+    }
+}
+```
+
+You can switch to a new working directory using this message (note that the directory must be in `allowed_directories`):
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "selectdir",
+        "directory": ...string...
+    }
+}
+```
+
+Overwriting the current list of excluded file patterns is possible with this message:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "exclude",
+        "exclude": ...array of strings...
+    }
+}
+```
+
+### TerminalEventHandler
+
+Writing to the terminal:
+```
+{ 
+    "key": "shell",
+    "data":
+    {
+        "command": "write",
+        "value": ...string...
+    }
+}
+```
+
+You can read terminal command history like so:
+```
+{ 
+    "key": "shell",
+    "data":
+    {
+        "command": "read",
+        "count": ...number...
+    }
+}
+```
+
+### ProcessManagingEventHandler
+
+Starting, stopping and restarting supervisor processes can be done using similar messages (where `command` is `start`, `stop` or `restart`):
+```
+{ 
+    "key": "processmanager",
+    "data":
+    {
+        "command": ...string...,
+        "process_name": ...string...
+    }
+}
+```
+
+### LogMonitoringEventHandler
+
+To change which supervisor process is monitored use this message:
+```
+{ 
+    "key": "logmonitor",
+    "data" :
+    {
+        "command": "process_name",
+        "value": ...string...
+    }
+}
+```
+
+To set the tail length of logs (the monitor will send back the last `value` characters of the log):
+```
+{ 
+    "key": "logmonitor",
+    "data" :
+    {
+        "command": "log_tail",
+        "value": ...number...
+    }
+}
+```
+
+### FSMManagingEventHandler
+
+To attempt executing a trigger on the FSM use (this will also generate an FSM update message):
+```
+{
+    "key": "fsm",
+    "data" :
+    {
+        "command": "trigger",
+        "value": ...string...
+    }
+}
+```
+
+To force the broadcasting of an FSM update you can use this message:
+```
+{
+    "key": "fsm",
+    "data" :
+    {
+        "command": "update"
+    }
+}
+```
+
+This event handler broadcasts FSM update messages after handling commands in the following format:
+```
+{
+    "key": "fsm_update",
+    "data" :
+    {
+        "current_state": ...string...,
+        "valid_transitions": ...array of {"trigger": ...string...} objects...
+    }
+}
+```
+

VERSION

@@ -1 +1 @@
-egyptianmau
+mainecoon

bashrc

@@ -9,7 +9,7 @@ _precmd()
 {
     [ -z "$AT_PROMPT" ] && return
     unset AT_PROMPT
-    history -a
+    [ -z "$TFW_DELAY_HISTAPPEND" ] && history -a
 }
 trap _precmd DEBUG
 

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = baseimage-tutorial-framework
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -0,0 +1,36 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+set SPHINXPROJ=baseimage-tutorial-framework
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

docs/source/components/components.rst

@@ -0,0 +1,25 @@
+Components
+----------
+
+.. automodule:: tfw.components
+
+.. autoclass:: IdeEventHandler
+    :members:
+
+.. autoclass:: TerminalEventHandler
+    :members:
+
+.. autoclass:: ProcessManagingEventHandler
+    :members:
+
+.. autoclass:: LogMonitoringEventHandler
+    :members:
+
+.. autoclass:: TerminalCommands
+    :members:
+
+.. autoclass:: HistoryMonitor
+    :members:
+
+.. autoclass:: BashMonitor
+    :members:

docs/source/conf.py

@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+sys.path.insert(0, os.path.abspath('../../lib'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'baseimage-tutorial-framework'
+copyright = '2018, Avatao Innovative Learning Kft'
+author = 'Kristóf Tóth'
+
+# The short X.Y version
+version = ''
+# The full version, including alpha/beta/rc tags
+release = 'mainecoon'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path .
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'baseimage-tutorial-frameworkdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'baseimage-tutorial-framework.tex', 'baseimage-tutorial-framework Documentation',
+     'Kristóf Tóth', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'baseimage-tutorial-framework', 'baseimage-tutorial-framework Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'baseimage-tutorial-framework', 'baseimage-tutorial-framework Documentation',
+     author, 'baseimage-tutorial-framework', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Extension configuration -------------------------------------------------
+def skip(app, what, name, obj, skip, options):
+    if name == "__init__":
+        return False
+    return skip
+
+def setup(app):
+    app.connect("autodoc-skip-member", skip)

docs/source/foundations/eventhandlers.rst

@@ -0,0 +1,12 @@
+Event handler base classes
+--------------------------
+
+Subclass these to create your cusom event handlers.
+
+.. automodule:: tfw
+
+.. autoclass:: EventHandlerBase
+    :members:
+
+.. autoclass:: TriggeredEventHandler
+    :members:

docs/source/foundations/fsms.rst

@@ -0,0 +1,12 @@
+FSM base classes
+----------------
+
+Subclass these to create an FSM that fits your tutorial/challenge.
+
+.. automodule:: tfw
+
+.. autoclass:: FSMBase
+    :members:
+
+.. autoclass:: LinearFSM
+    :members:

docs/source/foundations/tfwserver.rst

@@ -0,0 +1,7 @@
+TFWServer
+---------
+
+.. automodule:: tfw.networking
+
+.. autoclass:: TFWServer
+    :members:

docs/source/index.rst

@@ -0,0 +1,45 @@
+.. baseimage-tutorial-framework documentation master file, created by
+   sphinx-quickstart on Fri Jun  1 14:29:07 2018.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to baseimage-tutorial-framework's documentation!
+========================================================
+
+Foundations
+-----------
+
+This part covers the soil which the framework is based on and stuff you will need to develop your own challenges.
+
+.. toctree::
+    :glob:
+
+    foundations/*
+
+Networking
+----------
+
+You can use these to send messages to the frontend or the event handlers through TFWServer.
+
+.. toctree::
+    :glob:
+
+    networking/*
+
+Components
+----------
+
+These are pre-written components for you to use, such as our IDE, terminal or console.
+
+.. toctree::
+    :glob:
+
+    components/*
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/source/networking/networking.rst

@@ -0,0 +1,17 @@
+Networking
+----------
+
+.. automodule:: tfw.networking
+
+.. autoclass:: TFWServerConnector
+    :members:
+
+.. automodule:: tfw.networking.event_handlers
+
+.. autoclass:: ServerUplinkConnector
+    :members:
+
+.. automodule:: tfw.networking
+
+.. autoclass:: MessageSender
+    :members:

lib/envvars.py

@@ -1,12 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-from collections import namedtuple
-from os import environ
-
-
-def prefixed_envvars_to_namedtuple(prefix: str, tuple_name: str):
-    envvars = {envvar.replace(prefix, '', 1): environ.get(envvar)
-               for envvar in environ.keys()
-               if envvar.startswith(prefix)}
-    return namedtuple(tuple_name, envvars)(**envvars)

lib/envvars/__init__.py

@@ -0,0 +1,25 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from collections import namedtuple
+from os import environ
+
+from tfw.decorators import lazy_property
+
+
+class LazyEnvironment:
+    def __init__(self, prefix, tuple_name):
+        self._prefix = prefix
+        self._tuple_name = tuple_name
+
+    @lazy_property
+    def environment(self):
+        return self.prefixed_envvars_to_namedtuple()
+
+    def prefixed_envvars_to_namedtuple(self):
+        envvars = {
+            envvar.replace(self._prefix, '', 1): environ.get(envvar)
+            for envvar in environ.keys()
+            if envvar.startswith(self._prefix)
+        }
+        return namedtuple(self._tuple_name, envvars)(**envvars)

lib/tao/config/envvars.py

@@ -1,6 +1,6 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from envvars import prefixed_envvars_to_namedtuple
+from envvars import LazyEnvironment
 
-TAOENV = prefixed_envvars_to_namedtuple('AVATAO_', 'taoenvtuple')
+TAOENV = LazyEnvironment('AVATAO_', 'taoenvtuple').environment

lib/tfw/__init__.py

@@ -1,6 +1,7 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from .event_handler_base import EventHandlerBase, TriggeredEventHandler
+from .event_handler_base import EventHandlerBase, FSMAwareEventHandler, BroadcastingEventHandler
 from .fsm_base import FSMBase
 from .linear_fsm import LinearFSM
+from .yaml_fsm import YamlFSM

lib/tfw/components/__init__.py

@@ -7,3 +7,6 @@ from .terminal_event_handler import TerminalEventHandler
 from .ide_event_handler import IdeEventHandler
 from .history_monitor import HistoryMonitor, BashMonitor, GDBMonitor
 from .terminal_commands import TerminalCommands
+from .log_monitoring_event_handler import LogMonitoringEventHandler
+from .fsm_managing_event_handler import FSMManagingEventHandler
+from .snapshot_provider import SnapshotProvider

lib/tfw/components/directory_monitor.py

@@ -15,10 +15,11 @@ LOG = logging.getLogger(__name__)
 
 
 class DirectoryMonitor(ObserverMixin):
-    def __init__(self, directory):
-        ObserverMixin.__init__(self)
-        self.eventhandler = IdeReloadWatchdogEventHandler()
-        self.observer.schedule(self.eventhandler, directory, recursive=True)
+    def __init__(self, ide_key, directories):
+        self.eventhandler = IdeReloadWatchdogEventHandler(ide_key)
+        for directory in directories:
+            self.observer.schedule(self.eventhandler, directory, recursive=True)
+
         self.pause, self.resume = self.eventhandler.pause, self.eventhandler.resume
 
     @property
@@ -43,8 +44,9 @@ class DirectoryMonitor(ObserverMixin):
 
 
 class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler):
-    def __init__(self):
+    def __init__(self, ide_key):
         super().__init__()
+        self.ide_key = ide_key
         self.uplink = ServerUplinkConnector()
         self._paused = False
         self.ignore = 0
@@ -63,8 +65,10 @@ class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler):
             self.ignore = self.ignore - 1
             return
         LOG.debug(event)
-        self.uplink.send({'key': 'ide',
-                          'data': {'command': 'reload'}})
+        self.uplink.send({
+            'key': self.ide_key,
+            'data': {'command': 'reload'}
+        })
 
 
 def with_monitor_paused(fun):

lib/tfw/components/directory_monitoring_event_handler.py

@@ -15,12 +15,19 @@ class DirectoryMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin):
     def __init__(self, key, directory):
         super().__init__(key)
         self._directory = directory
-        MonitorManagerMixin.__init__(self, DirectoryMonitor, self._directory)
+        MonitorManagerMixin.__init__(
+            self,
+            DirectoryMonitor,
+            key,
+            self._directory
+        )
 
-        self.commands = {'pause':     self.pause,
-                         'resume':    self.resume,
-                         'ignore':    self.ignore,
-                         'selectdir': self.selectdir}
+        self.commands = {
+            'pause':     self.pause,
+            'resume':    self.resume,
+            'ignore':    self.ignore,
+            'selectdir': self.selectdir
+        }
 
     @property
     def directory(self):

lib/tfw/components/fsm_managing_event_handler.py

@@ -0,0 +1,68 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from tfw import EventHandlerBase
+from tfw.crypto import KeyManager, sign_message
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class FSMManagingEventHandler(EventHandlerBase):
+    def __init__(self, key, fsm_type):
+        super().__init__(key)
+        self.fsm = fsm_type()
+        self._fsm_updater = FSMUpdater(self.fsm)
+        self.auth_key = KeyManager().auth_key
+
+        self.command_handlers = {
+            'trigger': self.handle_trigger,
+            'update':  self.handle_update
+        }
+
+    def handle_event(self, message):
+        try:
+            data = message['data']
+            message['data'] = self.command_handlers[data['command']](data)
+            fsm_update_message = self._fsm_updater.generate_fsm_update()
+            sign_message(self.auth_key, message)
+            sign_message(self.auth_key, fsm_update_message)
+            self.server_connector.broadcast(fsm_update_message)
+            return message
+        except KeyError:
+            LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
+
+    def handle_trigger(self, data):
+        trigger = data['value']
+        self.fsm.step(trigger)
+        return data
+
+    def handle_update(self, data):
+        # pylint: disable=no-self-use
+        return data
+
+
+class FSMUpdater:
+    def __init__(self, fsm):
+        self.fsm = fsm
+
+    def generate_fsm_update(self):
+        return {
+            'key': 'fsm_update',
+            'data': self.get_fsm_state_and_transitions()
+        }
+
+    def get_fsm_state_and_transitions(self):
+        state = self.fsm.state
+        valid_transitions = [
+            {'trigger': trigger}
+            for trigger in self.fsm.get_triggers(self.fsm.state)
+        ]
+        last_trigger = self.fsm.trigger_history[-1] if self.fsm.trigger_history else None
+        in_accepted_state = state in self.fsm.accepted_states
+        return {
+            'current_state': state,
+            'valid_transitions': valid_transitions,
+            'last_trigger': last_trigger,
+            'in_accepted_state': in_accepted_state
+        }

lib/tfw/components/history_monitor.py

@@ -36,15 +36,17 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC):
     See examples below.
     """
     def __init__(self, histfile):
-        CallbackMixin.__init__(self)
-        ObserverMixin.__init__(self)
         self.histfile = histfile
         self._history = []
         self._last_length = len(self._history)
-        self.observer.schedule(CallbackEventHandler([self.histfile],
-                                                    self._fetch_history,
-                                                    self._invoke_callbacks),
-                               dirname(self.histfile))
+        self.observer.schedule(
+            CallbackEventHandler(
+                [self.histfile],
+                self._fetch_history,
+                self._invoke_callbacks
+            ),
+            dirname(self.histfile)
+        )
 
     @property
     def history(self):
@@ -55,7 +57,10 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC):
         with open(self.histfile, 'r') as ifile:
             pattern = compileregex(self.command_pattern)
             data = ifile.read()
-            self._history = [self.sanitize_command(command) for command in findall(pattern, data)]
+            self._history = [
+                self.sanitize_command(command)
+                for command in findall(pattern, data)
+            ]
 
     @property
     @abstractmethod
@@ -76,10 +81,10 @@ class BashMonitor(HistoryMonitor):
     HistoryMonitor for monitoring bash CLI sessions.
     This requires the following to be set in bash
     (note that this is done automatically by TFW):
-        PROMPT_COMMAND="history -a"
-        shopt -s cmdhist
-        shopt -s histappend
-        unset HISTCONTROL
+    PROMPT_COMMAND="history -a"
+    shopt -s cmdhist
+    shopt -s histappend
+    unset HISTCONTROL
     """
     @property
     def command_pattern(self):

lib/tfw/components/ide_event_handler.py

@@ -4,7 +4,7 @@
 from os.path import isfile, join, relpath, exists, isdir, realpath
 from glob import glob
 from fnmatch import fnmatchcase
-from collections import Iterable
+from typing import Iterable
 
 from tfw import EventHandlerBase
 from tfw.mixins import MonitorManagerMixin
@@ -41,8 +41,8 @@ class FileManager:  # pylint: disable=too-many-instance-attributes
     def workdir(self, directory):
         if not exists(directory) or not isdir(directory):
             raise EnvironmentError(f'"{directory}" is not a directory!')
-        if not self._is_in_whitelisted_dir(directory):
-            raise EnvironmentError(f'Directory "{directory}" is not in whitelist!')
+        if not self._is_in_allowed_dir(directory):
+            raise EnvironmentError(f'Directory "{directory}" is not allowed!')
         self._workdir = directory
 
     @property
@@ -65,8 +65,13 @@ class FileManager:  # pylint: disable=too-many-instance-attributes
 
     @property
     def files(self):
-        return [self._relpath(file) for file in glob(join(self._workdir, '**/*'), recursive=True)
-                if isfile(file) and self._is_in_whitelisted_dir(file) and not self._is_blacklisted(file)]
+        return [
+            self._relpath(file)
+            for file in glob(join(self._workdir, '**/*'), recursive=True)
+            if isfile(file)
+            and self._is_in_allowed_dir(file)
+            and not self._is_blacklisted(file)
+        ]
 
     @property
     def file_contents(self):
@@ -78,11 +83,17 @@ class FileManager:  # pylint: disable=too-many-instance-attributes
         with open(self._filepath(self.filename), 'w', errors='surrogateescape') as ofile:
             ofile.write(value)
 
-    def _is_in_whitelisted_dir(self, path):
-        return any(realpath(path).startswith(allowed_dir) for allowed_dir in self.allowed_directories)
+    def _is_in_allowed_dir(self, path):
+        return any(
+            realpath(path).startswith(allowed_dir)
+            for allowed_dir in self.allowed_directories
+        )
 
     def _is_blacklisted(self, file):
-        return any(fnmatchcase(file, blacklisted) for blacklisted in self.exclude)
+        return any(
+            fnmatchcase(file, blacklisted)
+            for blacklisted in self.exclude
+        )
 
     def _filepath(self, filename):
         return join(self._workdir, filename)
@@ -92,43 +103,61 @@ class FileManager:  # pylint: disable=too-many-instance-attributes
 
 
 class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
-    # pylint: disable=too-many-arguments
+    # pylint: disable=too-many-arguments,anomalous-backslash-in-string
     """
     Event handler implementing the backend of our browser based IDE.
     By default all files in the directory specified in __init__ are displayed
     on the fontend. Note that this is a stateful component.
 
-    This EventHandler accepts messages that have a data["command"] key specifying
+    When any file in the selected directory changes they are automatically refreshed
+    on the frontend (this is done by listening to inotify events).
+
+    This EventHandler accepts messages that have a data['command'] key specifying
     a command to be executed.
-    The API of each command is documented in their respective handlers.
+
+    The API of each command is documented in their respective handler.
     """
     def __init__(self, key, directory, allowed_directories, selected_file=None, exclude=None):
         """
         :param key: the key this instance should listen to
         :param directory: working directory which the EventHandler should serve files from
-        :param allowed_directories: list of directories that can be switched to using the selectdir command
+        :param allowed_directories: list of directories that can be switched to using selectdir
         :param selected_file: file that is selected by default
-        :param exclude: list of filenames that should not appear between files (for *.o, *.pyc, etc.)
+        :param exclude: list of filenames that should not appear between files (for .o, .pyc, etc.)
         """
         super().__init__(key)
         try:
-            self.filemanager = FileManager(allowed_directories=allowed_directories, working_directory=directory,
-                                           selected_file=selected_file, exclude=exclude)
+            self.filemanager = FileManager(
+                allowed_directories=allowed_directories,
+                working_directory=directory,
+                selected_file=selected_file,
+                exclude=exclude
+            )
         except IndexError:
-            raise EnvironmentError(f'No file(s) in IdeEventHandler working_directory "{directory}"!')
-        MonitorManagerMixin.__init__(self, DirectoryMonitor, self.filemanager.workdir)
+            raise EnvironmentError(
+                f'No file(s) in IdeEventHandler working_directory "{directory}"!'
+            )
 
-        self.commands = {'read':      self.read,
-                         'write':     self.write,
-                         'select':    self.select,
-                         'selectdir': self.select_dir,
-                         'exclude':   self.exclude}
+        MonitorManagerMixin.__init__(
+            self,
+            DirectoryMonitor,
+            self.key,
+            self.filemanager.allowed_directories
+        )
+
+        self.commands = {
+            'read':      self.read,
+            'write':     self.write,
+            'select':    self.select,
+            'selectdir': self.select_dir,
+            'exclude':   self.exclude
+        }
 
     def read(self, data):
         """
         Read the currently selected file.
 
-        :return: message with the contents of the file in data['content']
+        :return dict: message with the contents of the file in data['content']
         """
         try:
             data['content'] = self.filemanager.file_contents
@@ -144,8 +173,9 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
         """
         Overwrites a file with the desired string.
 
-        :param data: TFW message data containing keys:
-               |-string: containing the desired file contents
+        :param data: TFW message data containing key 'content'
+                     (new file content)
+
         """
         self.monitor.ignore = self.monitor.ignore + 1
         try:
@@ -159,8 +189,8 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
         """
         Selects a file from the current directory.
 
-        :param data: TFW message data containing keys:
-               |-filename: name of file to select relative to the current directory
+        :param data: TFW message data containing 'filename'
+                     (name of file to select relative to the current directory)
         """
         try:
             self.filemanager.filename = data['filename']
@@ -172,10 +202,10 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
         """
         Select a new working directory to display files from.
 
-        :param data: TFW message data containing keys:
-               |-directory: absolute path of diretory to select.
-                            must be a path whitelisted in
-                            self.allowed_directories
+        :param data: TFW message data containing 'directory'
+                     (absolute path of diretory to select.
+                     must be a path whitelisted in
+                     self.allowed_directories)
         """
         try:
             self.filemanager.workdir = data['directory']
@@ -193,8 +223,9 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
         """
         Overwrite list of excluded files
 
-        :param data: TFW message data containing keys:
-               |-exclude: list of filename patterns to be excluded, e.g.: ["*.pyc", "*.o"]
+        :param data: TFW message data containing 'exclude'
+                     (list of unix-style filename patterns to be excluded,
+                     e.g.: ["\*.pyc", "\*.o")
         """
         try:
             self.filemanager.exclude = list(data['exclude'])

lib/tfw/components/log_monitor.py

@@ -0,0 +1,56 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+import logging
+from os.path import dirname
+
+from watchdog.events import PatternMatchingEventHandler as PatternMatchingWatchdogEventHandler
+
+from tfw.networking.event_handlers import ServerUplinkConnector
+from tfw.decorators import RateLimiter
+from tfw.mixins import ObserverMixin, SupervisorLogMixin
+
+
+class LogMonitor(ObserverMixin):
+    def __init__(self, process_name, log_tail=0):
+        self.prevent_log_recursion()
+        event_handler = SendLogWatchdogEventHandler(
+            process_name,
+            log_tail=log_tail
+        )
+        self.observer.schedule(
+            event_handler,
+            event_handler.path
+        )
+
+    @staticmethod
+    def prevent_log_recursion():
+        # This is done to prevent inotify event logs triggering themselves (infinite log recursion)
+        logging.getLogger('watchdog.observers.inotify_buffer').propagate = False
+
+
+class SendLogWatchdogEventHandler(PatternMatchingWatchdogEventHandler, SupervisorLogMixin):
+    def __init__(self, process_name, log_tail=0):
+        self.process_name = process_name
+        self.procinfo = self.supervisor.getProcessInfo(self.process_name)
+        super().__init__([
+            self.procinfo['stdout_logfile'],
+            self.procinfo['stderr_logfile']
+        ])
+        self.uplink = ServerUplinkConnector()
+        self.log_tail = log_tail
+
+    @property
+    def path(self):
+        return dirname(self.procinfo['stdout_logfile'])
+
+    @RateLimiter(rate_per_second=5)
+    def on_modified(self, event):
+        self.uplink.send({
+            'key': 'processlog',
+            'data': {
+                'command': 'new_log',
+                'stdout': self.read_stdout(self.process_name, tail=self.log_tail),
+                'stderr': self.read_stderr(self.process_name, tail=self.log_tail)
+            }
+        })

lib/tfw/components/log_monitoring_event_handler.py

@@ -0,0 +1,67 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from tfw import EventHandlerBase
+from tfw.mixins import MonitorManagerMixin
+from tfw.config.logs import logging
+from .log_monitor import LogMonitor
+
+LOG = logging.getLogger(__name__)
+
+
+class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin):
+    """
+    Monitors the output of a supervisor process (stdout, stderr) and
+    sends the results to the frontend.
+
+    Accepts messages that have a data['command'] key specifying
+    a command to be executed.
+
+    The API of each command is documented in their respective handler.
+    """
+    def __init__(self, key, process_name, log_tail=0):
+        super().__init__(key)
+        self.process_name = process_name
+        self.log_tail = log_tail
+        MonitorManagerMixin.__init__(
+            self,
+            LogMonitor,
+            self.process_name,
+            self.log_tail
+        )
+
+        self.command_handlers = {
+            'process_name': self.handle_process_name,
+            'log_tail':     self.handle_log_tail
+        }
+
+    def handle_event(self, message):
+        try:
+            data = message['data']
+            self.command_handlers[data['command']](data)
+            self.reload_monitor()
+        except KeyError:
+            LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
+
+    def handle_process_name(self, data):
+        """
+        Changes the monitored process.
+
+        :param data: TFW message data containing 'value'
+                     (name of the process to monitor)
+        """
+        self.set_monitor_args(data['value'], self.log_tail)
+
+    def handle_log_tail(self, data):
+        """
+        Sets tail length of the log the monitor will send
+        to the frontend (the monitor will send back the last
+        'value' characters of the log).
+
+        :param data: TFW message data containing 'value'
+                     (new tail length)
+        """
+        self.set_monitor_args(self.process_name, data['value'])
+
+    def cleanup(self):
+        self.monitor.stop()

lib/tfw/components/process_managing_event_handler.py

@@ -4,19 +4,20 @@
 from xmlrpc.client import Fault as SupervisorFault
 
 from tfw import EventHandlerBase
-from tfw.mixins import SupervisorMixin
+from tfw.mixins import SupervisorMixin, SupervisorLogMixin
 from tfw.config.logs import logging
 from .directory_monitor import with_monitor_paused
 
 LOG = logging.getLogger(__name__)
 
 
-class ProcessManager(SupervisorMixin):
+class ProcessManager(SupervisorMixin, SupervisorLogMixin):
     def __init__(self):
-        self.commands = {'start': self.start_process,
-                         'stop': self.stop_process,
-                         'restart': self.restart_process,
-                         'readlog': self.read_log}
+        self.commands = {
+            'start':   self.start_process,
+            'stop':    self.stop_process,
+            'restart': self.restart_process
+        }
 
     def __call__(self, command, process_name):
         return self.commands[command](process_name)
@@ -26,30 +27,38 @@ class ProcessManagingEventHandler(EventHandlerBase):
     """
     Event handler that can manage processes managed by supervisor.
 
-    This EventHandler accepts messages that have a data["command"] key specifying
+    This EventHandler accepts messages that have a data['command'] key specifying
     a command to be executed.
-    Every message must contain a data["process_name"] field with the name of the
+    Every message must contain a data['process_name'] field with the name of the
     process to manage. This is the name specified in supervisor config files like so:
     [program:someprogram]
 
     Commands available: start, stop, restart, readlog
     (the names are as self-documenting as it gets)
     """
-    def __init__(self, key, dirmonitor=None):
+    def __init__(self, key, dirmonitor=None, log_tail=0):
         super().__init__(key)
-        self.key = key
         self.monitor = dirmonitor
         self.processmanager = ProcessManager()
+        self.log_tail = log_tail
 
     @with_monitor_paused
     def handle_event(self, message):
         try:
             data = message['data']
-            self.processmanager(data['command'], data['process_name'])
+            try:
+                self.processmanager(data['command'], data['process_name'])
+            except SupervisorFault as fault:
+                message['data']['error'] = fault.faultString
+            finally:
+                message['data']['stdout'] = self.processmanager.read_stdout(
+                    data['process_name'],
+                    self.log_tail
+                )
+                message['data']['stderr'] = self.processmanager.read_stderr(
+                    data['process_name'],
+                    self.log_tail
+                )
             return message
         except KeyError:
             LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
-        except SupervisorFault as fault:
-            message['data']['error'] = fault.faultString
-            message['data']['log'] = self.processmanager.read_log(message['data']['process_name'])
-            return message

lib/tfw/components/snapshot_provider.py

@@ -0,0 +1,175 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+import re
+from subprocess import run, CalledProcessError
+from getpass import getuser
+from os.path import isdir
+from datetime import datetime
+from uuid import uuid4
+
+
+class SnapshotProvider:
+    def __init__(self, directory, git_dir):
+        self._classname = self.__class__.__name__
+        author = f'{getuser()} via TFW {self._classname}'
+        self.gitenv = {
+            'GIT_DIR': git_dir,
+            'GIT_WORK_TREE': directory,
+            'GIT_AUTHOR_NAME': author,
+            'GIT_AUTHOR_EMAIL': '',
+            'GIT_COMMITTER_NAME': author,
+            'GIT_COMMITTER_EMAIL': '',
+            'GIT_PAGER': 'cat'
+        }
+
+        self._init_repo()
+        self.__last_valid_branch = self._branch
+
+    def _init_repo(self):
+        self._check_environment()
+
+        if not self._repo_is_initialized:
+            self._run(('git', 'init'))
+
+        if self._number_of_commits == 0:
+            try:
+                self._snapshot()
+            except CalledProcessError:
+                raise EnvironmentError(f'{self._classname} cannot init on empty directories!')
+
+        self._check_head_not_detached()
+
+    def _check_environment(self):
+        if not isdir(self.gitenv['GIT_DIR']) or not isdir(self.gitenv['GIT_WORK_TREE']):
+            raise EnvironmentError(f'{self._classname}: "directory" and "git_dir" must exist!')
+
+    @property
+    def _repo_is_initialized(self):
+        return self._run(
+            ('git', 'status'),
+            check=False
+        ).returncode == 0
+
+    @property
+    def _number_of_commits(self):
+        return int(
+            self._get_stdout((
+                'git', 'rev-list',
+                '--all',
+                '--count'
+            ))
+        )
+
+    def _snapshot(self):
+        self._run((
+            'git', 'add',
+            '-A'
+        ))
+        self._run((
+            'git', 'commit',
+            '-m', 'Snapshot'
+        ))
+
+    def _check_head_not_detached(self):
+        if self._head_detached:
+            raise EnvironmentError(f'{self._classname} cannot init from detached HEAD state!')
+
+    @property
+    def _head_detached(self):
+        return self._branch == 'HEAD'
+
+    @property
+    def _branch(self):
+        return self._get_stdout((
+            'git', 'rev-parse',
+            '--abbrev-ref', 'HEAD'
+        ))
+
+    def _get_stdout(self, *args, **kwargs):
+        kwargs['capture_output'] = True
+        stdout_bytes = self._run(*args, **kwargs).stdout
+        return stdout_bytes.decode().rstrip('\n')
+
+    def _run(self, *args, **kwargs):
+        if 'check' not in kwargs:
+            kwargs['check'] = True
+        if 'env' not in kwargs:
+            kwargs['env'] = self.gitenv
+        return run(*args, **kwargs)
+
+    def take_snapshot(self):
+        if self._head_detached:
+            self._checkout_new_branch_from_head()
+        self._snapshot()
+
+    def _checkout_new_branch_from_head(self):
+        branch_name = uuid4()
+        self._run((
+            'git', 'branch',
+            branch_name
+        ))
+        self._checkout(branch_name)
+
+    def _checkout(self, what):
+        self._run((
+            'git', 'checkout',
+            what
+        ))
+
+    def restore_snapshot(self, date):
+        commit = self._get_commit_from_timestamp(date)
+        self._checkout(commit)
+
+    def _get_commit_from_timestamp(self, date):
+        return self._get_stdout((
+            'git', 'rev-list',
+            '--date=iso',
+            '-n', '1',
+            f'--before="{date.isoformat()}"',
+            self._last_valid_branch
+        ))
+
+    @property
+    def _last_valid_branch(self):
+        if not self._head_detached:
+            self.__last_valid_branch = self._branch
+        return self.__last_valid_branch
+
+    @property
+    def all_timelines(self):
+        return self._branches
+
+    @property
+    def _branches(self):
+        git_branch_output = self._get_stdout(('git', 'branch'))
+        regex_pattern = re.compile(r'(?:[^\S\n]|[*])')  # matches '*' and non-newline whitespace chars
+        return re.sub(regex_pattern, '', git_branch_output).splitlines()
+
+    @property
+    def timeline(self):
+        return self._last_valid_branch
+
+    @timeline.setter
+    def timeline(self, value):
+        self._checkout(value)
+
+    @property
+    def snapshots(self):
+        return self._pretty_log_branch()
+
+    def _pretty_log_branch(self):
+        git_log_output = self._get_stdout((
+            'git', 'log',
+            '--pretty=%H@%aI'
+        ))
+
+        commits = []
+        for line in git_log_output.splitlines():
+            commit_hash, timestamp = line.split('@')
+            commits.append({
+                'hash': commit_hash,
+                'timestamp': datetime.fromisoformat(timestamp)
+            })
+
+        return commits

lib/tfw/components/terminado_mini_server.py

@@ -14,15 +14,15 @@ LOG = logging.getLogger(__name__)
 class TerminadoMiniServer:
     def __init__(self, url, port, workdir, shellcmd):
         self.port = port
-        self._term_manager = SingleTermManager(shell_command=shellcmd,
-                                               term_settings={'cwd': workdir})
-        self.application = Application(
-            [(
-                url,
-                TerminadoMiniServer.ResetterTermSocket,
-                {'term_manager': self._term_manager}
-            )]
+        self._term_manager = SingleTermManager(
+            shell_command=shellcmd,
+            term_settings={'cwd': workdir}
         )
+        self.application = Application([(
+            url,
+            TerminadoMiniServer.ResetterTermSocket,
+            {'term_manager': self._term_manager}
+        )])
 
     @property
     def term_manager(self):
@@ -46,5 +46,10 @@ class TerminadoMiniServer:
 
 if __name__ == '__main__':
     LOG.info('Terminado Mini Server listening on %s', TFWENV.TERMINADO_PORT)
-    TerminadoMiniServer('/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, ['bash']).listen()
+    TerminadoMiniServer(
+        '/terminal',
+        TFWENV.TERMINADO_PORT,
+        TFWENV.TERMINADO_WD,
+        ['bash']
+    ).listen()
     IOLoop.instance().start()

lib/tfw/components/terminal_commands.py

@@ -3,6 +3,7 @@
 
 from abc import ABC
 from re import match
+from shlex import split
 
 from tfw.config.logs import logging
 
@@ -10,6 +11,7 @@ LOG = logging.getLogger(__name__)
 
 
 class TerminalCommands(ABC):
+    # pylint: disable=anomalous-backslash-in-string
     """
     A class you can use to define hooks for terminal commands. This means that you can
     have python code executed when the user enters a specific command to the terminal on
@@ -18,13 +20,13 @@ class TerminalCommands(ABC):
     To receive events you need to subscribe TerminalCommand.callback to a HistoryMonitor
     instance.
 
-    Inherit from this class and define methods which start with "command_". When the user
+    Inherit from this class and define methods which start with "command\_". When the user
     executes the command specified after the underscore, your method will be invoked. All
-    such commands must expect the parameter *args which will contain the arguments of the
+    such commands must expect the parameter \*args which will contain the arguments of the
     command.
 
     For example to define a method that runs when someone starts vim in the terminal
-    you have to define a method like: "def command_vim(self, *args)"
+    you have to define a method like: "def command_vim(self, \*args)"
 
     You can also use this class to create new commands similarly.
     """
@@ -35,8 +37,12 @@ class TerminalCommands(ABC):
             self._setup_bashrc_aliases(bashrc)
 
     def _build_command_to_implementation_dict(self):
-        return {self._parse_command_name(fun): getattr(self, fun) for fun in dir(self)
-                if callable(getattr(self, fun)) and self._is_command_implementation(fun)}
+        return {
+            self._parse_command_name(fun): getattr(self, fun)
+            for fun in dir(self)
+            if callable(getattr(self, fun))
+            and self._is_command_implementation(fun)
+        }
 
     def _setup_bashrc_aliases(self, bashrc):
         with open(bashrc, 'a') as ofile:
@@ -57,10 +63,10 @@ class TerminalCommands(ABC):
         return match(self._command_method_regex, string)
 
     def callback(self, history):
-        parts = history[-1].split()
+        parts = split(history[-1])
         command = parts[0]
         if command in self.command_implemetations.keys():
             try:
                 self.command_implemetations[command](*parts[1:])
-            except IndexError:
-                LOG.debug('Command "%s" failed!', command)
+            except Exception: # pylint: disable=broad-except
+                LOG.exception('Command "%s" failed:', command)

lib/tfw/components/terminal_event_handler.py

@@ -16,9 +16,9 @@ class TerminalEventHandler(EventHandlerBase):
     sessions to connect to. You need to instanciate this in order for frontend
     terminals to work.
 
-    This EventHandler accepts messages that have a data["command"] key specifying
+    This EventHandler accepts messages that have a data['command'] key specifying
     a command to be executed.
-    The API of each command is documented in their respective handlers.
+    The API of each command is documented in their respective handler.
     """
     def __init__(self, key, monitor):
         """
@@ -29,9 +29,19 @@ class TerminalEventHandler(EventHandlerBase):
         self.working_directory = TFWENV.TERMINADO_DIR
         self._historymonitor = monitor
         bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash']
-        self.terminado_server = TerminadoMiniServer('/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, bash_as_user_cmd)
-        self.commands = {'write': self.write,
-                         'read': self.read}
+
+        self.terminado_server = TerminadoMiniServer(
+            '/terminal',
+            TFWENV.TERMINADO_PORT,
+            TFWENV.TERMINADO_WD,
+            bash_as_user_cmd
+        )
+
+        self.commands = {
+            'write': self.write,
+            'read': self.read
+        }
+
         if self._historymonitor:
             self._historymonitor.watch()
         self.terminado_server.listen()
@@ -54,18 +64,19 @@ class TerminalEventHandler(EventHandlerBase):
         Writes a string to the terminal session (on the pty level).
         Useful for pre-typing and executing commands for the user.
 
-        :param data: TFW message data containing keys:
-               |-shellcmd: command to be written to the pty
+        :param data: TFW message data containing 'value'
+                     (command to be written to the pty)
         """
-        self.terminado_server.pty.write(data['shellcmd'])
+        self.terminado_server.pty.write(data['value'])
+        return data
 
     def read(self, data):
         """
         Reads the history of commands executed.
 
-        :param data: TFW message data containing keys:
-               |-count: the number of history elements to return
-        :return: message with list of commands in data['history']
+        :param data: TFW message data containing 'count'
+                     (the number of history elements to return)
+        :return dict: message with list of commands in data['history']
         """
         data['count'] = int(data.get('count', 1))
         if self.historymonitor:

lib/tfw/config/envvars.py

@@ -1,6 +1,6 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from envvars import prefixed_envvars_to_namedtuple
+from envvars import LazyEnvironment
 
-TFWENV = prefixed_envvars_to_namedtuple('TFW_', 'tfwenvtuple')
+TFWENV = LazyEnvironment('TFW_', 'tfwenvtuple').environment

lib/tfw/crypto.py

@@ -0,0 +1,107 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from functools import wraps
+from base64 import b64encode, b64decode
+from copy import deepcopy
+from hashlib import md5
+from os import urandom, chmod
+from os.path import exists
+from stat import S_IRUSR, S_IWUSR, S_IXUSR
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.hashes import SHA256
+from cryptography.hazmat.primitives.hmac import HMAC as _HMAC
+from cryptography.exceptions import InvalidSignature
+
+from tfw.networking import message_bytes
+from tfw.decorators import lazy_property
+from tfw.config import TFWENV
+
+
+def message_checksum(message):
+    return md5(message_bytes(message)).hexdigest()
+
+
+def sign_message(key, message):
+    signature = message_signature(key, message)
+    message['signature'] = b64encode(signature).decode()
+
+
+def message_signature(key, message):
+    return HMAC(key, message_bytes(message)).signature
+
+
+def verify_message(key, message):
+    message = deepcopy(message)
+    try:
+        signature_b64 = message.pop('signature')
+        signature = b64decode(signature_b64)
+        actual_signature = message_signature(key, message)
+        return signature == actual_signature
+    except KeyError:
+        return False
+
+
+class KeyManager:
+    def __init__(self):
+        self.keyfile = TFWENV.AUTH_KEY
+        if not exists(self.keyfile):
+            self._init_auth_key()
+
+    @lazy_property
+    def auth_key(self):
+        with open(self.keyfile, 'rb') as ifile:
+            return ifile.read()
+
+    def _init_auth_key(self):
+        key = self.generate_key()
+        with open(self.keyfile, 'wb') as ofile:
+            ofile.write(key)
+        self._chmod_700_keyfile()
+        return key
+
+    @staticmethod
+    def generate_key():
+        return urandom(32)
+
+    def _chmod_700_keyfile(self):
+        chmod(self.keyfile, S_IRUSR | S_IWUSR | S_IXUSR)
+
+
+class HMAC:
+    def __init__(self, key, message):
+        self.key = key
+        self.message = message
+        self._hmac = _HMAC(
+            key=key,
+            algorithm=SHA256(),
+            backend=default_backend()
+        )
+
+    def _reload_if_finalized(f):
+        # pylint: disable=no-self-argument,not-callable
+        @wraps(f)
+        def wrapped(instance, *args, **kwargs):
+            if getattr(instance, '_finalized', False):
+                instance.__init__(instance.key, instance.message)
+            ret_val = f(instance, *args, **kwargs)
+            setattr(instance, '_finalized', True)
+            return ret_val
+        return wrapped
+
+    @property
+    @_reload_if_finalized
+    def signature(self):
+        self._hmac.update(self.message)
+        signature = self._hmac.finalize()
+        return signature
+
+    @_reload_if_finalized
+    def verify(self, signature):
+        self._hmac.update(self.message)
+        try:
+            self._hmac.verify(signature)
+            return True
+        except InvalidSignature:
+            return False

lib/tfw/decorators/__init__.py

@@ -2,3 +2,4 @@
 # All Rights Reserved. See LICENSE file for details.
 
 from .rate_limiter import RateLimiter
+from .lazy_property import lazy_property

lib/tfw/decorators/lazy_property.py

@@ -0,0 +1,21 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from functools import update_wrapper
+
+
+class lazy_property:
+    """
+    Decorator that replaces a function with the value
+    it calculates on the first call.
+    """
+    def __init__(self, func):
+        self.func = func
+        update_wrapper(self, func)
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self  # avoids potential __new__ TypeError
+        value = self.func(instance)
+        setattr(instance, self.func.__name__, value)
+        return value

lib/tfw/event_handler_base.py

@@ -3,8 +3,11 @@
 
 from abc import ABC, abstractmethod
 
-from tfw.networking import deserialize_tfw_msg
 from tfw.networking.event_handlers import ServerConnector
+from tfw.crypto import message_checksum, KeyManager, verify_message
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
 
 
 class EventHandlerBase(ABC):
@@ -16,30 +19,51 @@ class EventHandlerBase(ABC):
     """
     def __init__(self, key):
         self.server_connector = ServerConnector()
-        self.key = key
-        self.subscribe(self.key, 'reset')
+        self.keys = [key]
+        self.subscribe(*self.keys)
         self.server_connector.register_callback(self.event_handler_callback)
 
-    def event_handler_callback(self, msg_parts):
+    @property
+    def key(self):
+        """
+        Returns the oldest key this EventHandler was subscribed to.
+        """
+        return self.keys[0]
+
+    def event_handler_callback(self, message):
         """
         Callback that is invoked when receiving a message.
         Dispatches messages to handler methods and sends
         a response back in case the handler returned something.
         This is subscribed in __init__().
         """
-        message = deserialize_tfw_msg(*msg_parts)
+        if not self.check_key(message):
+            return
+
         response = self.dispatch_handling(message)
         if response:
-            response['key'] = message['key']
             self.server_connector.send(response)
 
+    def check_key(self, message):
+        """
+        Checks whether the message is intended for this
+        EventHandler.
+
+        This is necessary because ZMQ handles PUB - SUB
+        connetions with pattern matching (e.g. someone
+        subscribed to 'fsm' will receive 'fsm_update'
+        messages as well.
+        """
+        return message['key'] in self.keys
+
     def dispatch_handling(self, message):
         """
         Used to dispatch messages to their specific handlers.
+
+        :param message: the message received
+        :returns: the message to send back
         """
-        if message['key'] != 'reset':
-            return self.handle_event(message)
-        return self.handle_reset(message)
+        return self.handle_event(message)
 
     @abstractmethod
     def handle_event(self, message):
@@ -47,29 +71,22 @@ class EventHandlerBase(ABC):
         Abstract method that implements the handling of messages.
 
         :param message: the message received
+        :returns: the message to send back
         """
         raise NotImplementedError
 
-    def handle_reset(self, message):
-        # pylint: disable=unused-argument,no-self-use
-        """
-        Usually 'reset' events receive some sort of special treatment.
-
-        :param message: the message received
-        """
-        return None
-
     def subscribe(self, *keys):
         """
         Subscribe this EventHandler to receive events for given keys.
         Note that you can subscribe to the same key several times in which
         case you will need to unsubscribe multiple times in order to stop
         receiving events.
-        
+
         :param keys: list of keys to subscribe to
         """
         for key in keys:
             self.server_connector.subscribe(key)
+            self.keys.append(key)
 
     def unsubscribe(self, *keys):
         """
@@ -79,6 +96,7 @@ class EventHandlerBase(ABC):
         """
         for key in keys:
             self.server_connector.unsubscribe(key)
+            self.keys.remove(key)
 
     def cleanup(self):
         """
@@ -88,17 +106,62 @@ class EventHandlerBase(ABC):
         pass
 
 
-class TriggeredEventHandler(EventHandlerBase, ABC):
+class FSMAwareEventHandler(EventHandlerBase, ABC):
     # pylint: disable=abstract-method
     """
-    Abstract base class for EventHandlers which are only triggered in case
-    TFWServer has successfully triggered an FSM step defined in __init__.
+    Abstract base class for EventHandlers which automatically
+    keep track of the state of the TFW FSM.
     """
-    def __init__(self, key, trigger):
+    def __init__(self, key):
         super().__init__(key)
-        self.trigger = trigger
+        self.subscribe('fsm_update')
+        self.fsm_state = None
+        self.in_accepted_state = False
+        self._auth_key = KeyManager().auth_key
 
     def dispatch_handling(self, message):
-        if message.get('trigger') == self.trigger:
-            return super().dispatch_handling(message)
-        return None
+        if message['key'] == 'fsm_update':
+            if verify_message(self._auth_key, message):
+                self._handle_fsm_update(message)
+            return None
+        return super().dispatch_handling(message)
+
+    def _handle_fsm_update(self, message):
+        try:
+            new_state = message['data']['current_state']
+            trigger = message['data']['last_trigger']
+            if self.fsm_state != new_state:
+                self.handle_fsm_step(self.fsm_state, new_state, trigger)
+            self.fsm_state = new_state
+            self.in_accepted_state = message['data']['in_accepted_state']
+        except KeyError:
+            LOG.error('Invalid fsm_update message received!')
+
+    def handle_fsm_step(self, from_state, to_state, trigger):
+        """
+        Called in case the TFW FSM has stepped.
+        """
+        pass
+
+
+class BroadcastingEventHandler(EventHandlerBase, ABC):
+    # pylint: disable=abstract-method
+    """
+    Abstract base class for EventHandlers which broadcast responses
+    and intelligently ignore their own broadcasted messages they receive.
+    """
+    def __init__(self, key):
+        super().__init__(key)
+        self.own_message_hashes = []
+
+    def event_handler_callback(self, message):
+        message_hash = message_checksum(message)
+
+        if message_hash in self.own_message_hashes:
+            self.own_message_hashes.remove(message_hash)
+            return
+
+        response = self.dispatch_handling(message)
+        if response:
+            self.own_message_hashes.append(message_checksum(response))
+            self.server_connector.broadcast(response)

lib/tfw/fsm_base.py

@@ -1,36 +1,66 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from typing import List
+from collections import defaultdict
 
-from transitions import Machine
+from transitions import Machine, MachineError
 
 from tfw.mixins import CallbackMixin
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
 
 
-class FSMBase(CallbackMixin):
+class FSMBase(Machine, CallbackMixin):
     """
     A general FSM base class you can inherit from to track user progress.
     See linear_fsm.py for an example use-case.
-    TFW the transitions library for state machines, please refer to their
+    TFW uses the transitions library for state machines, please refer to their
     documentation for more information on creating your own machines:
     https://github.com/pytransitions/transitions
     """
     states, transitions = [], []
 
-    def __init__(self, initial: str = None, accepted_states: List[str] = None):
-        CallbackMixin.__init__(self)
-        self.accepted_states = accepted_states or [self.states[-1]]
-        self.machine = Machine(model=self,
-                               states=self.states,
-                               transitions=self.transitions,
-                               initial=initial or self.states[0],
-                               send_event=True,
-                               ignore_invalid_triggers=True,
-                               after_state_change='execute_callbacks')
+    def __init__(self, initial=None, accepted_states=None):
+        self.accepted_states = accepted_states or [self.states[-1].name]
+        self.trigger_predicates = defaultdict(list)
+        self.trigger_history = []
+
+        Machine.__init__(
+            self,
+            states=self.states,
+            transitions=self.transitions,
+            initial=initial or self.states[0],
+            send_event=True,
+            ignore_invalid_triggers=True,
+            after_state_change='execute_callbacks'
+        )
 
     def execute_callbacks(self, event_data):
         self._execute_callbacks(event_data.kwargs)
 
     def is_solved(self):
-        return self.state in self.accepted_states # pylint: disable=no-member
+        return self.state in self.accepted_states  # pylint: disable=no-member
+
+    def subscribe_predicate(self, trigger, *predicates):
+        self.trigger_predicates[trigger].extend(predicates)
+
+    def unsubscribe_predicate(self, trigger, *predicates):
+        self.trigger_predicates[trigger] = [
+            predicate
+            for predicate in self.trigger_predicates[trigger]
+            not in predicates
+        ]
+
+    def step(self, trigger):
+        predicate_results = (
+            predicate()
+            for predicate in self.trigger_predicates[trigger]
+        )
+
+        if all(predicate_results):
+            try:
+                self.trigger(trigger)
+                self.trigger_history.append(trigger)
+            except (AttributeError, MachineError):
+                LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)

lib/tfw/linear_fsm.py

@@ -1,6 +1,8 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
+from transitions import State
+
 from .fsm_base import FSMBase
 
 
@@ -11,12 +13,19 @@ class LinearFSM(FSMBase):
     a number of steps specified in the constructor. It automatically sets up 2
     actions (triggers) between states as such:
     (0) --  step_1  --> (1) --  step_2  --> (2) --  step_3  --> (3) ... and so on
-        \-step_next-/       \-step_next-/       \-step_next-/
     """
     def __init__(self, number_of_steps):
-        self.states = list(map(str, range(number_of_steps)))
+        self.states = [State(name=str(index)) for index in range(number_of_steps)]
         self.transitions = []
-        for index in self.states[:-1]:
-            self.transitions.append({'trigger': f'step_{int(index)+1}', 'source': index, 'dest': str(int(index)+1)})
-            self.transitions.append({'trigger': 'step_next', 'source': index, 'dest': str(int(index)+1)})
+        for state in self.states[:-1]:
+            self.transitions.append({
+                'trigger': f'step_{int(state.name)+1}',
+                'source': state.name,
+                'dest': str(int(state.name)+1)
+            })
+            self.transitions.append({
+                'trigger': 'step_next',
+                'source': state.name,
+                'dest': str(int(state.name)+1)
+            })
         super(LinearFSM, self).__init__()

lib/tfw/mixins/__init__.py

@@ -1,7 +1,7 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from .supervisor_mixin import SupervisorMixin
+from .supervisor_mixin import SupervisorMixin, SupervisorLogMixin
 from .callback_mixin import CallbackMixin
 from .observer_mixin import ObserverMixin
 from .monitor_manager_mixin import MonitorManagerMixin

lib/tfw/mixins/callback_mixin.py

@@ -3,21 +3,33 @@
 
 from functools import partial
 
+from tfw.decorators import lazy_property
+
 
 class CallbackMixin:
-    def __init__(self):
-        self._callbacks = []
+    @lazy_property
+    def _callbacks(self):
+        # pylint: disable=no-self-use
+        return []
 
     def subscribe_callback(self, callback, *args, **kwargs):
         """
         Subscribe a callable to invoke once an event is triggered.
         :param callback: callable to be executed on events
-        :param *args: arguments passed to callable
-        :param **kwargs: kwargs passed to callable
+        :param args: arguments passed to callable
+        :param kwargs: kwargs passed to callable
         """
         fun = partial(callback, *args, **kwargs)
         self._callbacks.append(fun)
 
+    def subscribe_callbacks(self, *callbacks):
+        """
+        Subscribe a list of callbacks to incoke once an event is triggered.
+        :param callbacks: callbacks to be subscribed
+        """
+        for callback in callbacks:
+            self.subscribe_callback(callback)
+
     def unsubscribe_callback(self, callback):
         self._callbacks.remove(callback)
 

lib/tfw/mixins/monitor_manager_mixin.py

@@ -7,21 +7,24 @@ LOG = logging.getLogger(__name__)
 
 
 class MonitorManagerMixin:
-    def __init__(self, monitor_type, directory):
+    def __init__(self, monitor_type, *monitor_args):
         self._monitor_type = monitor_type
         self._monitor = None
-        self._monitored_directory = directory
+        self.monitor_args = monitor_args
         self.reload_monitor()
 
     @property
     def monitor(self):
         return self._monitor
 
+    def set_monitor_args(self, *monitor_args):
+        self.monitor_args = monitor_args
+
     def reload_monitor(self):
         if self._monitor:
             try:
                 self._monitor.stop()
             except KeyError:
                 LOG.debug('Working directory was removed – ignoring...')
-        self._monitor = self._monitor_type(self._monitored_directory)
+        self._monitor = self._monitor_type(*self.monitor_args)
         self._monitor.watch()   # This runs on a separate thread

lib/tfw/mixins/observer_mixin.py

@@ -3,10 +3,14 @@
 
 from watchdog.observers import Observer
 
+from tfw.decorators import lazy_property
+
 
 class ObserverMixin:
-    def __init__(self):
-        self.observer = Observer()
+    @lazy_property
+    def observer(self):
+        # pylint: disable=no-self-use
+        return Observer()
 
     def watch(self):
         self.observer.start()

lib/tfw/mixins/supervisor_mixin.py

@@ -6,12 +6,18 @@ from xmlrpc.client import Fault as SupervisorFault
 from contextlib import suppress
 from os import remove
 
+from tfw.decorators import lazy_property
 from tfw.config import TFWENV
 
 
-class SupervisorMixin:
-    supervisor = xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor
+class SupervisorBaseMixin:
+    @lazy_property
+    def supervisor(self):
+        # pylint: disable=no-self-use
+        return xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor
 
+
+class SupervisorMixin(SupervisorBaseMixin):
     def stop_process(self, process_name):
         with suppress(SupervisorFault):
             self.supervisor.stopProcess(process_name)
@@ -19,12 +25,20 @@ class SupervisorMixin:
     def start_process(self, process_name):
         self.supervisor.startProcess(process_name)
 
-    def read_log(self, process_name):
-        logs = self.supervisor.readProcessStderrLog(process_name, 0, 0)
-        remove(self.supervisor.getProcessInfo(process_name)['stderr_logfile'])
-        self.supervisor.clearProcessLogs(process_name)
-        return logs
-
     def restart_process(self, process_name):
         self.stop_process(process_name)
         self.start_process(process_name)
+
+
+class SupervisorLogMixin(SupervisorBaseMixin):
+    def read_stdout(self, process_name, tail=0):
+        return self.supervisor.readProcessStdoutLog(process_name, -tail, 0)
+
+    def read_stderr(self, process_name, tail=0):
+        return self.supervisor.readProcessStderrLog(process_name, -tail, 0)
+
+    def clear_logs(self, process_name):
+        for logfile in ('stdout_logfile', 'stderr_logfile'):
+            with suppress(FileNotFoundError):
+                remove(self.supervisor.getProcessInfo(process_name)[logfile])
+        self.supervisor.clearProcessLogs(process_name)

lib/tfw/networking/__init__.py

@@ -1,9 +1,9 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from .serialization import serialize_tfw_msg, deserialize_tfw_msg, validate_message
+from .serialization import serialize_tfw_msg, deserialize_tfw_msg
+from .serialization import with_deserialize_tfw_msg, message_bytes
 from .zmq_connector_base import ZMQConnectorBase
-# from .controller_connector import ControllerConnector # TODO: readd once controller stuff is resolved
 from .message_sender import MessageSender
 from .event_handlers.server_connector import ServerUplinkConnector as TFWServerConnector
 from .server.tfw_server import TFWServer

lib/tfw/networking/controller_connector.py

@@ -1,18 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-import zmq
-from zmq.eventloop.zmqstream import ZMQStream
-
-from tfw.config import TFWENV
-from tfw.networking import ZMQConnectorBase
-
-
-class ControllerConnector(ZMQConnectorBase):
-    def __init__(self, zmq_context=None):
-        super(ControllerConnector, self).__init__(zmq_context)
-        self._zmq_rep_socket = self._zmq_context.socket(zmq.REP)
-        self._zmq_rep_socket.connect(f'tcp://localhost:{TFWENV.CONTROLLER_PORT}')
-        self._zmq_rep_stream = ZMQStream(self._zmq_rep_socket)
-
-        self.register_callback = self._zmq_rep_stream.on_recv_stream

lib/tfw/networking/event_handlers/server_connector.py

@@ -6,9 +6,12 @@ from functools import partial
 import zmq
 from zmq.eventloop.zmqstream import ZMQStream
 
-from tfw.networking import serialize_tfw_msg
+from tfw.networking import serialize_tfw_msg, with_deserialize_tfw_msg
 from tfw.networking import ZMQConnectorBase
 from tfw.config import TFWENV
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
 
 
 class ServerDownlinkConnector(ZMQConnectorBase):
@@ -20,7 +23,10 @@ class ServerDownlinkConnector(ZMQConnectorBase):
 
         self.subscribe = partial(self._zmq_sub_socket.setsockopt_string, zmq.SUBSCRIBE)
         self.unsubscribe = partial(self._zmq_sub_socket.setsockopt_string, zmq.UNSUBSCRIBE)
-        self.register_callback = self._zmq_sub_stream.on_recv
+
+    def register_callback(self, callback):
+        callback = with_deserialize_tfw_msg(callback)
+        self._zmq_sub_stream.on_recv(callback)
 
 
 class ServerUplinkConnector(ZMQConnectorBase):
@@ -34,22 +40,40 @@ class ServerUplinkConnector(ZMQConnectorBase):
 
     def send_to_eventhandler(self, message):
         """
-        Send a message to an event handler.
+        Send a message to an event handler through the TFW server.
+
+        This envelopes the desired message in the 'data' field of the message to
+        TFWServer, which will mirror it to event handlers.
+
         :param message: JSON message you want to send
-        :param message['key']: key of event handler you want to address
         """
-        nested_message = {'key': message['key'], 'data': message.pop('data')}
-        message['key'] = 'mirror'
-        message['data'] = nested_message
-        self.send(message)
+        self.send({
+            'key': 'mirror',
+            'data': message
+        })
 
     def send(self, message):
         """
-        Send a message to the TFW server
+        Send a message to the frontend through the TFW server.
+
         :param message: JSON message you want to send
         """
         self._zmq_push_socket.send_multipart(serialize_tfw_msg(message))
 
+    def broadcast(self, message):
+        """
+        Broadast a message through the TFW server.
+
+        This envelopes the desired message in the 'data' field of the message to
+        TFWServer, which will broadast it.
+
+        :param message: JSON message you want to send
+        """
+        self.send({
+            'key': 'broadcast',
+            'data': message
+        })
+
 
 class ServerConnector(ServerUplinkConnector, ServerDownlinkConnector):
     pass

lib/tfw/networking/message_sender.py

@@ -1,30 +1,54 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from datetime import datetime
-
 from tfw.networking.event_handlers import ServerUplinkConnector
 
 
 class MessageSender:
     """
-    Provides a mechanism to send messages to our frontend messaging component which
-    displays messages with the key "message".
+    Provides mechanisms to send messages to our frontend messaging component.
     """
-    def __init__(self, custom_key: str = None):
+    def __init__(self):
         self.server_connector = ServerUplinkConnector()
-        self.key = custom_key or 'message'
+        self.key = 'message'
+        self.queue_key = 'queueMessages'
 
     def send(self, originator, message):
         """
-        Sends a message to the key specified in __init__.
+        Sends a message.
         :param originator: name of sender to be displayed on the frontend
         :param message: message to send
         """
         data = {
             'originator': originator,
-            'timestamp': datetime.now().isoformat(),
             'message': message
         }
-        self.server_connector.send({'key': self.key,
-                                    'data': data})
+        self.server_connector.send({
+            'key': self.key,
+            'data': data
+        })
+
+    def queue_messages(self, originator, messages):
+        """
+        Queues a list of messages to be displayed in a chatbot-like manner.
+        :param originator: name of sender to be displayed on the frontend
+        :param messages: list of messages to queue
+        """
+        data = {
+            'messages': [
+                {'message': message, 'originator': originator}
+                for message in messages
+            ]
+        }
+        self.server_connector.send({
+            'key': self.queue_key,
+            'data': data
+        })
+
+    @staticmethod
+    def generate_messages_from_queue(queue_message):
+        for message in queue_message['data']['messages']:
+            yield {
+                'key': 'message',
+                'data': message
+            }

lib/tfw/networking/serialization.py

@@ -22,35 +22,64 @@ The purpose of this module is abstracting away this low level behaviour.
 """
 
 import json
-
-
-def validate_message(message):
-    return 'key' in message
+from functools import wraps
 
 
 def serialize_tfw_msg(message):
+    """
+    Create TFW multipart data from message dict
+    """
     return _serialize_all(message['key'], message)
 
 
+def with_deserialize_tfw_msg(fun):
+    @wraps(fun)
+    def wrapper(message_parts):
+        message = deserialize_tfw_msg(*message_parts)
+        return fun(message)
+    return wrapper
+
+
 def deserialize_tfw_msg(*args):
+    """
+    Return message from TFW multipart data
+    """
     return _deserialize_all(*args)[1]
 
 
 def _serialize_all(*args):
-    return tuple(_serialize_single(arg) for arg in args)
+    return tuple(
+        _serialize_single(arg)
+        for arg in args
+    )
 
 
 def _deserialize_all(*args):
-    return tuple(_deserialize_single(arg) for arg in args)
+    return tuple(
+        _deserialize_single(arg)
+        for arg in args
+    )
 
 
 def _serialize_single(data):
+    """
+    Return input as bytes
+    (serialize input if it is JSON)
+    """
     if not isinstance(data, str):
-        data = json.dumps(data)
+        data = message_bytes(data)
     return _encode_if_needed(data)
 
 
+def message_bytes(message):
+    return json.dumps(message, sort_keys=True).encode()
+
+
 def _deserialize_single(data):
+    """
+    Try parsing input as JSON, return it as
+    string if parsing fails.
+    """
     try:
         return json.loads(data)
     except ValueError:
@@ -58,12 +87,20 @@ def _deserialize_single(data):
 
 
 def _encode_if_needed(value):
+    """
+    Return input as bytes
+    (encode if input is string)
+    """
     if isinstance(value, str):
         value = value.encode('utf-8')
     return value
 
 
 def _decode_if_needed(value):
+    """
+    Return input as string
+    (decode if input is bytes)
+    """
     if isinstance(value, (bytes, bytearray)):
         value = value.decode('utf-8')
     return value

lib/tfw/networking/server/__init__.py

@@ -3,5 +3,3 @@
 
 from .event_handler_connector import EventHandlerConnector, EventHandlerUplinkConnector, EventHandlerDownlinkConnector
 from .tfw_server import TFWServer
-from .zmq_websocket_handler import ZMQWebSocketProxy
-# from .controller_responder import ControllerResponder  # TODO: readd once controller stuff is resolved

lib/tfw/networking/server/controller_responder.py

@@ -1,38 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-from tfw.networking import deserialize_all, serialize_all, ControllerConnector
-
-
-class ControllerResponder:
-    def __init__(self, fsm):
-        self.fsm = fsm
-        self.token = None
-        self.controller_connector = ControllerConnector()
-        self.controller_connector.register_callback(self.handle_controller_request)
-        self.controller_request_handlers = {
-            'solution_check': self.handle_solution_check_request,
-            'test': self.handle_test_request,
-            'token': self.handle_token_request
-        }
-
-    def handle_controller_request(self, stream, msg_parts):
-        key, data = deserialize_all(*msg_parts)
-        response = self.controller_request_handlers[key](data)
-        stream.send_multipart(serialize_all(self.token, response))
-
-    def handle_test_request(self, data):
-        # pylint: disable=unused-argument,no-self-use
-        return 'OK'
-
-    def handle_token_request(self, data):
-        if self.token is None:
-            self.token = data
-        return {'token': self.token}
-
-    def handle_solution_check_request(self, data):
-        # pylint: disable=unused-argument
-        return {
-            'solved': self.fsm.is_solved(),
-            'message': 'solved' if self.fsm.is_solved() else 'not solved'
-        }

lib/tfw/networking/server/event_handler_connector.py

@@ -4,7 +4,7 @@
 import zmq
 from zmq.eventloop.zmqstream import ZMQStream
 
-from tfw.networking import ZMQConnectorBase, serialize_tfw_msg
+from tfw.networking import ZMQConnectorBase, serialize_tfw_msg, with_deserialize_tfw_msg
 from tfw.config import TFWENV
 from tfw.config.logs import logging
 
@@ -32,6 +32,7 @@ class EventHandlerUplinkConnector(ZMQConnectorBase):
 
 class EventHandlerConnector(EventHandlerDownlinkConnector, EventHandlerUplinkConnector):
     def register_callback(self, callback):
+        callback = with_deserialize_tfw_msg(callback)
         self._zmq_pull_stream.on_recv(callback)
 
     def send_message(self, message: dict):

lib/tfw/networking/server/tfw_server.py

@@ -1,15 +1,16 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from collections import defaultdict
+from abc import ABC, abstractmethod
+from contextlib import suppress
 
 from tornado.web import Application
 
-from tfw.networking import MessageSender
 from tfw.networking.event_handlers import ServerUplinkConnector
 from tfw.networking.server import EventHandlerConnector
+from tfw.networking import MessageSender
 from tfw.config.logs import logging
-from .zmq_websocket_handler import ZMQWebSocketProxy
+from .zmq_websocket_proxy import ZMQWebSocketProxy
 
 LOG = logging.getLogger(__name__)
 
@@ -18,103 +19,79 @@ class TFWServer:
     """
     This class handles the proxying of messages between the frontend and event handers.
     It proxies messages from the "/ws" route to all event handlers subscribed to a ZMQ
-    SUB socket. It also manages an FSM you can define as a constructor argument.
+    SUB socket.
     """
-    def __init__(self, fsm_type):
-        """
-        :param fsm_type: the type of FSM you want TFW to use
-        """
-        self._fsm = fsm_type()
-        self._fsm_updater = FSMUpdater(self._fsm)
-        self._fsm_manager = FSMManager(self._fsm)
-        self._fsm.subscribe_callback(self._fsm_updater.update)
+    def __init__(self):
         self._event_handler_connector = EventHandlerConnector()
+        self._uplink_connector = ServerUplinkConnector()
 
-        self.application = Application(
-            [(r'/ws', ZMQWebSocketProxy, {'make_eventhandler_message': self.make_eventhandler_message,
-                                          'proxy_filter': self.proxy_filter,
-                                          'handle_trigger': self.handle_trigger,
-                                          'event_handler_connector': self._event_handler_connector})]
-        )
-        #self.controller_responder = ControllerResponder(self.fsm) TODO: add this once controller stuff is resolved
+        self.application = Application([(
+            r'/ws', ZMQWebSocketProxy, {
+                'event_handler_connector': self._event_handler_connector,
+                'message_handlers': [self.handle_trigger, self.handle_recover],
+                'frontend_message_handlers': [self.save_frontend_messages]
+            })])
 
-    @property
-    def fsm(self):
-        return self._fsm
-
-    @property
-    def fsm_manager(self):
-        return self._fsm_manager
-
-    def make_eventhandler_message(self, message):
-        self.trigger_fsm(message)
-        message['FSMUpdate'] = self._fsm_updater.get_fsm_state_and_transitions()
-        return message
+        self._frontend_messages = FrontendMessageStorage()
 
     def handle_trigger(self, message):
-        LOG.debug('Executing handler for trigger "%s"', message.get('trigger', ''))
-        self.trigger_fsm(message)
+        if 'trigger' in message:
+            LOG.debug('Executing handler for trigger "%s"', message.get('trigger', ''))
+            self._uplink_connector.send_to_eventhandler({
+                'key': 'fsm',
+                'data': {
+                    'command': 'trigger',
+                    'value': message.get('trigger', '')
+                }
+            })
 
-    def trigger_fsm(self, message):
-        trigger = message.get('trigger', '')
-        try:
-            self._fsm_manager.trigger(trigger, message)
-        except AttributeError:
-            LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
+    def handle_recover(self, message):
+        if message['key'] == 'recover':
+            self._frontend_messages.replay_messages(self._uplink_connector)
+            self._frontend_messages.clear()
 
-    def proxy_filter(self, message):
-        # pylint: disable=unused-argument,no-self-use
-        return True
+    def save_frontend_messages(self, message):
+        self._frontend_messages.save_message(message)
 
     def listen(self, port):
         self.application.listen(port)
 
 
-class FSMManager:
-    def __init__(self, fsm):
-        self._fsm = fsm
-        self.trigger_predicates = defaultdict(list)
-        self.messenge_sender = MessageSender()
+class MessageStorage(ABC):
+    def __init__(self):
+        self.saved_messages = []
 
-    @property
-    def fsm(self):
-        return self._fsm
+    def save_message(self, message):
+        with suppress(KeyError, AttributeError):
+            if self.filter_message(message):
+                self.saved_messages.extend(self.transform_message(message))
 
-    def trigger(self, trigger, message):
-        predicate_results = []
-        for predicate in self.trigger_predicates[trigger]:
-            success, message = predicate(message)
-            predicate_results.append(success)
-            self.messenge_sender.send('FSM', message)
+    @abstractmethod
+    def filter_message(self, message):
+        raise NotImplementedError
 
-        if all(predicate_results):
-            try:
-                self.fsm.trigger(trigger, message=message)
-            except AttributeError:
-                LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
+    def transform_message(self, message):   # pylint: disable=no-self-use
+        yield message
 
-    def subscribe_predicate(self, trigger, *predicates):
-        self.trigger_predicates[trigger].extend(predicates)
-
-    def unsubscribe_predicate(self, trigger, *predicates):
-        self.trigger_predicates[trigger] = [predicate for predicate in self.trigger_predicates[trigger]
-                                            not in predicates]
+    def clear(self):
+        self.saved_messages.clear()
 
 
-class FSMUpdater:
-    def __init__(self, fsm):
-        self.fsm = fsm
-        self.uplink = ServerUplinkConnector()
+class FrontendMessageStorage(MessageStorage):
+    def filter_message(self, message):
+        key = message['key']
+        command = message.get('data', {}).get('command')
+        return (
+            key in ('message', 'dashboard', 'queueMessages')
+            or key == 'ide' and command in ('select', 'read')
+        )
 
-    def update(self, kwargs_dict):
-        # pylint: disable=unused-argument
-        self.uplink.send(self.generate_fsm_update())
+    def transform_message(self, message):
+        if message['key'] == 'queueMessages':
+            yield from MessageSender.generate_messages_from_queue(message)
+        else:
+            yield message
 
-    def generate_fsm_update(self):
-        return {'key': 'FSMUpdate',
-                'data': self.get_fsm_state_and_transitions()}
-
-    def get_fsm_state_and_transitions(self):
-        state = self.fsm.state
-        valid_transitions = [{'trigger': trigger} for trigger in self.fsm.machine.get_triggers(self.fsm.state)]
-        return {'current_state': state, 'valid_transitions': valid_transitions}
+    def replay_messages(self, connector):
+        for message in self.saved_messages:
+            connector.send(message)

lib/tfw/networking/server/zmq_websocket_handler.py

@@ -1,91 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-import json
-from abc import ABC, abstractmethod
-
-from tornado.websocket import WebSocketHandler
-
-from tfw.networking import deserialize_tfw_msg, validate_message
-from tfw.config.logs import logging
-
-LOG = logging.getLogger(__name__)
-
-
-class ZMQWebSocketHandler(WebSocketHandler, ABC):
-    instances = set()
-
-    def initialize(self, **kwargs): # pylint: disable=arguments-differ
-        self._event_handler_connector = kwargs['event_handler_connector']
-
-    def prepare(self):
-        ZMQWebSocketHandler.instances.add(self)
-
-    def on_close(self):
-        ZMQWebSocketHandler.instances.remove(self)
-
-    def open(self, *args, **kwargs):
-        LOG.debug('WebSocket connection initiated')
-        self._event_handler_connector.register_callback(self.zmq_callback)
-
-    def zmq_callback(self, msg_parts):
-        keyhandlers = {'mirror': self.mirror}
-
-        message = deserialize_tfw_msg(*msg_parts)
-        LOG.debug('Received on pull socket: %s', message)
-        if not validate_message(message):
-            return
-
-        self.handle_trigger(message)
-        if message['key'] not in keyhandlers:
-            for instance in ZMQWebSocketHandler.instances:
-                instance.write_message(message)
-        else:
-            try:
-                keyhandlers[message['key']](message)
-            except KeyError:
-                LOG.error('Invalid mirror message format! Ignoring.')
-
-    def mirror(self, message):
-        message = message['data']
-        self._event_handler_connector.send_message(message)
-
-    def on_message(self, message):
-        LOG.debug('Received on WebSocket: %s', message)
-        if validate_message(message):
-            self.send_message(self.make_eventhandler_message(message))
-
-    @abstractmethod
-    def make_eventhandler_message(self, message):
-        raise NotImplementedError
-
-    def send_message(self, message: dict):
-        self._event_handler_connector.send_message(message)
-
-    @abstractmethod
-    def handle_trigger(self, message):
-        raise NotImplementedError
-
-    # much secure, very cors, wow
-    def check_origin(self, origin):
-        return True
-
-
-class ZMQWebSocketProxy(ZMQWebSocketHandler):
-    # pylint: disable=abstract-method
-    def initialize(self, **kwargs): # pylint: disable=arguments-differ
-        super(ZMQWebSocketProxy, self).initialize(**kwargs)
-        self._make_eventhandler_message = kwargs['make_eventhandler_message']
-        self._proxy_filter = kwargs['proxy_filter']
-        self._handle_trigger = kwargs['handle_trigger']
-
-    def on_message(self, message):
-        message = json.loads(message)
-        if self._proxy_filter(message):
-            super().on_message(message)
-
-    def make_eventhandler_message(self, message):
-        return self._make_eventhandler_message(message)
-
-    def handle_trigger(self, message):
-        self._handle_trigger(message)

lib/tfw/networking/server/zmq_websocket_proxy.py

@@ -0,0 +1,138 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+import json
+
+from tornado.websocket import WebSocketHandler
+
+from tfw.mixins import CallbackMixin
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class ZMQWebSocketProxy(WebSocketHandler):
+    # pylint: disable=abstract-method
+    instances = set()
+
+    def initialize(self, **kwargs): # pylint: disable=arguments-differ
+        self._event_handler_connector = kwargs['event_handler_connector']
+
+        self._message_handlers = kwargs.get('message_handlers', [])
+        self._frontend_message_handlers = kwargs.get('frontend_message_handlers', [])
+        self._eventhandler_message_handlers = kwargs.get('eventhandler_message_handlers', [])
+        self._proxy_filters = kwargs.get('proxy_filters', [])
+
+        self.proxy_eventhandler_to_websocket = TFWProxy(
+            self.send_eventhandler_message,
+            self.send_websocket_message
+        )
+        self.proxy_websocket_to_eventhandler = TFWProxy(
+            self.send_websocket_message,
+            self.send_eventhandler_message
+        )
+
+        self.subscribe_proxy_callbacks()
+
+    def subscribe_proxy_callbacks(self):
+        self.proxy_websocket_to_eventhandler.subscribe_proxy_callbacks_and_filters(
+            self._eventhandler_message_handlers + self._message_handlers,
+            self._proxy_filters
+        )
+
+        self.proxy_eventhandler_to_websocket.subscribe_proxy_callbacks_and_filters(
+            self._frontend_message_handlers + self._message_handlers,
+            self._proxy_filters
+        )
+
+    def prepare(self):
+        ZMQWebSocketProxy.instances.add(self)
+
+    def on_close(self):
+        ZMQWebSocketProxy.instances.remove(self)
+
+    def open(self, *args, **kwargs):
+        LOG.debug('WebSocket connection initiated')
+        self._event_handler_connector.register_callback(self.eventhander_callback)
+
+    def eventhander_callback(self, message):
+        """
+        Invoked on ZMQ messages from event handlers.
+        """
+        LOG.debug('Received on pull socket: %s', message)
+        self.proxy_eventhandler_to_websocket(message)
+
+    def on_message(self, message):
+        """
+        Invoked on WS messages from frontend.
+        """
+        message = json.loads(message)
+        LOG.debug('Received on WebSocket: %s', message)
+        self.proxy_websocket_to_eventhandler(message)
+
+    def send_eventhandler_message(self, message):
+        self._event_handler_connector.send_message(message)
+
+    @staticmethod
+    def send_websocket_message(message):
+        for instance in ZMQWebSocketProxy.instances:
+            instance.write_message(message)
+
+    # much secure, very cors, wow
+    def check_origin(self, origin):
+        return True
+
+
+class TFWProxy:
+    # pylint: disable=protected-access
+    def __init__(self, to_source, to_destination):
+        self.to_source = to_source
+        self.to_destination = to_destination
+
+        self.proxy_filters = CallbackMixin()
+        self.proxy_callbacks = CallbackMixin()
+
+        self.proxy_filters.subscribe_callback(self.validate_message)
+
+        self.keyhandlers = {
+            'mirror':    self.mirror,
+            'broadcast': self.broadcast
+        }
+
+    @staticmethod
+    def validate_message(message):
+        if 'key' not in message:
+            raise ValueError('Invalid TFW message format!')
+
+    def __call__(self, message):
+        try:
+            self.proxy_filters._execute_callbacks(message)
+        except ValueError:
+            LOG.exception('Invalid TFW message received!')
+            return
+
+        self.proxy_callbacks._execute_callbacks(message)
+
+        if message['key'] not in self.keyhandlers:
+            self.to_destination(message)
+        else:
+            handler = self.keyhandlers[message['key']]
+            try:
+                handler(message)
+            except KeyError:
+                LOG.error('Invalid "%s" message format! Ignoring.', handler.__name__)
+
+    def mirror(self, message):
+        message = message['data']
+        LOG.debug('Mirroring message: %s', message)
+        self.to_source(message)
+
+    def broadcast(self, message):
+        message = message['data']
+        LOG.debug('Broadcasting message: %s', message)
+        self.to_source(message)
+        self.to_destination(message)
+
+    def subscribe_proxy_callbacks_and_filters(self, proxy_callbacks, proxy_filters):
+        self.proxy_callbacks.subscribe_callbacks(*proxy_callbacks)
+        self.proxy_filters.subscribe_callbacks(*proxy_filters)

lib/tfw/yaml_fsm.py

@@ -0,0 +1,95 @@
+from subprocess import Popen, run
+from functools import partial, singledispatch
+from contextlib import suppress
+
+import yaml
+import jinja2
+from transitions import State
+
+from tfw import FSMBase
+
+
+class YamlFSM(FSMBase):
+    def __init__(self, config_file, jinja2_variables=None):
+        self.config = ConfigParser(config_file, jinja2_variables).config
+        self.setup_states()
+        super().__init__()  # FSMBase.__init__() requires states
+        self.setup_transitions()
+
+    def setup_states(self):
+        self.for_config_states_and_transitions_do(self.wrap_callbacks_with_subprocess_call)
+        self.states = [State(**state) for state in self.config['states']]
+
+    def setup_transitions(self):
+        self.for_config_states_and_transitions_do(self.subscribe_and_remove_predicates)
+        for transition in self.config['transitions']:
+            self.add_transition(**transition)
+
+    def for_config_states_and_transitions_do(self, what):
+        for array in ('states', 'transitions'):
+            for json_obj in self.config[array]:
+                what(json_obj)
+
+    @staticmethod
+    def wrap_callbacks_with_subprocess_call(json_obj):
+        topatch = ('on_enter', 'on_exit', 'prepare', 'before', 'after')
+        for key in json_obj:
+            if key in topatch:
+                json_obj[key] = partial(run_command_async, json_obj[key])
+
+    def subscribe_and_remove_predicates(self, json_obj):
+        if 'predicates' in json_obj:
+            for predicate in json_obj['predicates']:
+                self.subscribe_predicate(
+                    json_obj['trigger'],
+                    partial(
+                        command_statuscode_is_zero,
+                        predicate
+                        )
+                )
+
+        with suppress(KeyError):
+            json_obj.pop('predicates')
+
+
+def run_command_async(command, _):
+    Popen(command, shell=True)
+
+
+def command_statuscode_is_zero(command):
+    return run(command, shell=True).returncode == 0
+
+
+class ConfigParser:
+    def __init__(self, config_file, jinja2_variables):
+        self.read_variables = singledispatch(self._read_variables)
+        self.read_variables.register(dict, self._read_variables_dict)
+        self.read_variables.register(str, self._read_variables_str)
+
+        self.config = self.parse_config(config_file, jinja2_variables)
+
+    def parse_config(self, config_file, jinja2_variables):
+        config_string = self.read_file(config_file)
+        if jinja2_variables is not None:
+            variables = self.read_variables(jinja2_variables)
+            template = jinja2.Environment(loader=jinja2.BaseLoader).from_string(config_string)
+            config_string = template.render(**variables)
+        return yaml.safe_load(config_string)
+
+    @staticmethod
+    def read_file(filename):
+        with open(filename, 'r') as ifile:
+            return ifile.read()
+
+    @staticmethod
+    def _read_variables(variables):
+        raise TypeError(f'Invalid variables type {type(variables)}')
+
+    @staticmethod
+    def _read_variables_str(variables):
+        with open(variables, 'r') as ifile:
+            return yaml.safe_load(ifile)
+
+    @staticmethod
+    def _read_variables_dict(variables):
+        return variables

nginx/default.conf

@@ -4,6 +4,7 @@ server {
     proxy_connect_timeout 7d;
     proxy_send_timeout 7d;
     proxy_read_timeout 7d;
+    absolute_redirect off;
 
     location = /ws {
         proxy_pass http://127.0.0.1:${TFW_WEB_PORT};

requirements.txt

@@ -3,3 +3,6 @@ pyzmq==17.0.0
 transitions==0.6.4
 terminado==0.8.1
 watchdog==0.8.3
+PyYAML==3.12
+Jinja2==2.10
+cryptography==2.2.2

setup.py

@@ -0,0 +1,29 @@
+from os.path import dirname, realpath, join
+
+from setuptools import setup, find_packages
+
+here = dirname(realpath(__file__))
+
+with open(join(here, 'VERSION'), 'r') as ifile:
+    version = ifile.read().strip('\n')
+with open(join(here, 'requirements.txt'), 'r') as ifile:
+    requirements = ifile.read().splitlines()
+
+setup(
+    name = 'tfw',
+    version = version,
+    description = 'Avatao tutorial-framework',
+    url = 'https://github.com/avatao-content/baseimage-tutorial-framework',
+    author = 'Avatao.com Innovative Learning Kft.',
+    author_email = 'support@avatao.com',
+    license = 'custom',
+    packages = find_packages('lib'),
+    package_dir = {'': 'lib'},
+    install_requires = requirements,
+    extras_require = {
+        'docs': [
+            'sphinx >= 1.7.0',
+        ],
+    },
+    zip_safe = False,
+)

supervisor/components/tfw_server.conf

@@ -0,0 +1,4 @@
+[program:tfwserver]
+user=root
+directory=%(ENV_TFW_SERVER_DIR)s
+command=python3 tfw_server.py

supervisor/supervisord.conf

@@ -1,8 +1,10 @@
 [supervisord]
 user=root
-logfile = /tmp/supervisord.log
+logfile=/dev/null
+logfile_maxbytes=0
 loglevel = debug
 pidfile = /tmp/supervisord.pid
+strip_ansi=true
 
 [inet_http_server]
 port = 127.0.0.1:%(ENV_TFW_SUPERVISOR_HTTP_PORT)s

supervisor/tfw_server.py

@@ -0,0 +1,9 @@
+from tornado.ioloop import IOLoop
+
+from tfw.networking import TFWServer
+from tfw.config import TFWENV
+
+
+if __name__ == '__main__':
+    TFWServer().listen(TFWENV.WEB_PORT)
+    IOLoop.instance().start()