Merge branch 'pipeio'

Rate limit magic

.drone.yml

@@ -1,6 +1,6 @@
 pipeline:
   build:
-    image: eu.gcr.io/avatao-public/docker:el7
+    image: eu.gcr.io/avatao-challengestore/challenge-builder
     volumes:
       - /etc/docker:/etc/docker:ro
       - /root/.docker:/root/.docker:ro
@@ -10,4 +10,4 @@ pipeline:
       - docker push eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG}
     when:
       event: 'tag'
-      branch: refs/tags/egyptianmau-20*
+      branch: refs/tags/mainecoon-20*

.pylintrc

@@ -1,5 +1,5 @@
 [TYPECHECK]
 
 ignored-modules = zmq
-max-line-length = 150
-disable = missing-docstring, too-few-public-methods
+max-line-length = 120
+disable = missing-docstring, too-few-public-methods, invalid-name

Dockerfile

@@ -10,6 +10,7 @@ RUN curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash -
         supervisor                             \
         libzmq5                                \
         nginx                                  \
+        jq                                     \
         gettext-base                                                                                   &&\
     rm -rf /var/lib/apt/lists/*                                                                        &&\
     ln -sf /bin/bash /bin/sh
@@ -34,9 +35,11 @@ ENV PYTHONPATH="/usr/local/lib"                                            \
     TFW_NGINX_CONF="/etc/nginx/nginx.conf"                                 \
     TFW_NGINX_DEFAULT="/etc/nginx/sites-enabled/default"                   \
     TFW_NGINX_COMPONENTS="/etc/nginx/components"                           \
-    TFW_LIB_DIR="/usr/local/lib/"                                          \
+    TFW_LIB_DIR="/usr/local/lib"                                           \
     TFW_TERMINADO_DIR="/tmp/terminado_server"                              \
     TFW_FRONTEND_DIR="/srv/frontend"                                       \
+    TFW_SERVER_DIR="/srv/.tfw"                                             \
+    TFW_AUTH_KEY="/tmp/tfw-auth.key"                                       \
     TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history"                      \
     PROMPT_COMMAND="history -a"
 
@@ -45,13 +48,15 @@ RUN echo "export HISTFILE=${TFW_HISTFILE}" >> /tmp/bashrc &&\
     cat /tmp/bashrc >> /home/${AVATAO_USER}/.bashrc
 
 COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF}
+COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS}
 COPY nginx/nginx.conf ${TFW_NGINX_CONF}
 COPY nginx/default.conf ${TFW_NGINX_DEFAULT}
 COPY nginx/components/ ${TFW_NGINX_COMPONENTS}
-COPY lib LICENSE ${TFW_LIB_DIR}
+COPY lib LICENSE ${TFW_LIB_DIR}/
+COPY supervisor/tfw_server.py ${TFW_SERVER_DIR}/
 
-RUN for dir in "${TFW_LIB_DIR}"/{tfw,tao,envvars.py} "/etc/nginx" "/etc/supervisor"; do \
-        chown -R root:root "$dir" && chmod -R 700 "$dir";                               \
+RUN for dir in "${TFW_LIB_DIR}"/{tfw,tao,envvars} "/etc/nginx" "/etc/supervisor"; do \
+        chown -R root:root "$dir" && chmod -R 700 "$dir";                            \
     done
 
 ONBUILD ARG BUILD_CONTEXT="solvable"
@@ -63,7 +68,7 @@ ONBUILD COPY ${BUILD_CONTEXT}/supervisor/ ${TFW_SUPERVISORD_COMPONENTS}
 ONBUILD RUN for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do                               \
                 envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f~ && mv $f~ $f ;\
             done
-ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx"]
+ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx", "${TFW_LIB_DIR}/envvars", "${TFW_LIB_DIR}/tfw"]
 
 ONBUILD COPY ${BUILD_CONTEXT}/frontend /data/
 ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn install --frozen-lockfile || :

README.md

@@ -20,6 +20,20 @@ Frontend components use websockets to connect to the TFW server, to which you ca
 
 ![TFW architecture](docs/tfw_architecture.png)
 
+### Networking details
+
+Event handlers connect to the TFW server using ZMQ.
+They receive messages on their `SUB`(scribe) sockets, which are connected to the `PUB`(lish) socket of the server.
+Event handlers reply on their `PUSH` socket, then their messages are received on the `PULL` socket of the server.
+
+The TFW server is basically just a fancy proxy.
+It's behaviour is quite simple: it proxies every message received from the fontend to the event handlers and vice versa.
+
+The server is also capable of "mirroring" messages back to their source.
+This is useful for communication between event handlers or frontend components (event handler to event handler or frontend component to frontend component communication).
+
+Components can also broadcast messages (broadcasted messages are received both by event handlers and the frontend as well).
+
 ### Event handlers
 
 Imagine event handlers as callbacks that are invoked when TFW receives a specific type of message. For instance, you could send a message to the framework when the user does something of note.
@@ -75,9 +89,193 @@ The TFW message format:
 - The `data` object can contain anything you might want to send
 - The `trigger` key is an optional field that triggers an FSM action with that name from the current state (whatever that might be)
 
+To mirror messages back to their sources you can use a special messaging format, in which the message to be mirrored is enveloped inside the `data` field of the outer message:
+
+```text
+    "key": "mirror",
+    "data":
+    {
+        ...
+        The message you want to mirror (with it's own "key" and "data" fields)
+        ...
+    }
+```
+
+Broadcasting messages is possible in a similar manner by using `"key": "broadcast"` in the outer message.
 
 ## Where to go next
 
 Most of the components you need have docstrings included (hang on tight, this is work in progress) – refer to them for usage info.
 
-To get started you should take a look at the [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework) repository, which serves as an example project as well.
+In the `docs` folder you can find our Sphinx-based documentation, which you can build using the `hack/tfw.sh` script in the [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework) repository.
+
+To get started you should take a look at [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework), which serves as an example project as well.
+
+## API
+
+APIs exposed by our pre-witten event handlers are documented here.
+
+### IdeEventHandler
+
+You can read the content of the currently selected file like so:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "read"
+    }
+}
+```
+
+Use the following message to overwrite the content of the currently selected file:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "write",
+        "content": ...string...
+    }
+}
+```
+
+To select a file use the following message:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "select",
+        "filename": ...string...
+    }
+}
+```
+
+You can switch to a new working directory using this message (note that the directory must be in `allowed_directories`):
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "selectdir",
+        "directory": ...string...
+    }
+}
+```
+
+Overwriting the current list of excluded file patterns is possible with this message:
+```
+{ 
+    "key": "ide",
+    "data":
+    {
+        "command": "exclude",
+        "exclude": ...array of strings...
+    }
+}
+```
+
+### TerminalEventHandler
+
+Writing to the terminal:
+```
+{ 
+    "key": "shell",
+    "data":
+    {
+        "command": "write",
+        "value": ...string...
+    }
+}
+```
+
+You can read terminal command history like so:
+```
+{ 
+    "key": "shell",
+    "data":
+    {
+        "command": "read",
+        "count": ...number...
+    }
+}
+```
+
+### ProcessManagingEventHandler
+
+Starting, stopping and restarting supervisor processes can be done using similar messages (where `command` is `start`, `stop` or `restart`):
+```
+{ 
+    "key": "processmanager",
+    "data":
+    {
+        "command": ...string...,
+        "process_name": ...string...
+    }
+}
+```
+
+### LogMonitoringEventHandler
+
+To change which supervisor process is monitored use this message:
+```
+{ 
+    "key": "logmonitor",
+    "data" :
+    {
+        "command": "process_name",
+        "value": ...string...
+    }
+}
+```
+
+To set the tail length of logs (the monitor will send back the last `value` characters of the log):
+```
+{ 
+    "key": "logmonitor",
+    "data" :
+    {
+        "command": "log_tail",
+        "value": ...number...
+    }
+}
+```
+
+### FSMManagingEventHandler
+
+To attempt executing a trigger on the FSM use (this will also generate an FSM update message):
+```
+{
+    "key": "fsm",
+    "data" :
+    {
+        "command": "trigger",
+        "value": ...string...
+    }
+}
+```
+
+To force the broadcasting of an FSM update you can use this message:
+```
+{
+    "key": "fsm",
+    "data" :
+    {
+        "command": "update"
+    }
+}
+```
+
+This event handler broadcasts FSM update messages after handling commands in the following format:
+```
+{
+    "key": "fsm_update",
+    "data" :
+    {
+        "current_state": ...string...,
+        "valid_transitions": ...array of {"trigger": ...string...} objects...
+    }
+}
+```
+

VERSION

@@ -1 +1 @@
-egyptianmau
+mainecoon

bashrc

@@ -9,7 +9,7 @@ _precmd()
 {
     [ -z "$AT_PROMPT" ] && return
     unset AT_PROMPT
-    history -a
+    [ -z "$TFW_DELAY_HISTAPPEND" ] && history -a
 }
 trap _precmd DEBUG
 

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = baseimage-tutorial-framework
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -0,0 +1,36 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+set SPHINXPROJ=baseimage-tutorial-framework
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

docs/source/components/components.rst

@@ -0,0 +1,25 @@
+Components
+----------
+
+.. automodule:: tfw.components
+
+.. autoclass:: IdeEventHandler
+    :members:
+
+.. autoclass:: TerminalEventHandler
+    :members:
+
+.. autoclass:: ProcessManagingEventHandler
+    :members:
+
+.. autoclass:: LogMonitoringEventHandler
+    :members:
+
+.. autoclass:: TerminalCommands
+    :members:
+
+.. autoclass:: HistoryMonitor
+    :members:
+
+.. autoclass:: BashMonitor
+    :members:

docs/source/conf.py

@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+sys.path.insert(0, os.path.abspath('../../lib'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'baseimage-tutorial-framework'
+copyright = '2018, Avatao Innovative Learning Kft'
+author = 'Kristóf Tóth'
+
+# The short X.Y version
+version = ''
+# The full version, including alpha/beta/rc tags
+release = 'mainecoon'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path .
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'baseimage-tutorial-frameworkdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'baseimage-tutorial-framework.tex', 'baseimage-tutorial-framework Documentation',
+     'Kristóf Tóth', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'baseimage-tutorial-framework', 'baseimage-tutorial-framework Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'baseimage-tutorial-framework', 'baseimage-tutorial-framework Documentation',
+     author, 'baseimage-tutorial-framework', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Extension configuration -------------------------------------------------
+def skip(app, what, name, obj, skip, options):
+    if name == "__init__":
+        return False
+    return skip
+
+def setup(app):
+    app.connect("autodoc-skip-member", skip)

docs/source/foundations/eventhandlers.rst

@@ -0,0 +1,12 @@
+Event handler base classes
+--------------------------
+
+Subclass these to create your cusom event handlers.
+
+.. automodule:: tfw
+
+.. autoclass:: EventHandlerBase
+    :members:
+
+.. autoclass:: TriggeredEventHandler
+    :members:

docs/source/foundations/fsms.rst

@@ -0,0 +1,12 @@
+FSM base classes
+----------------
+
+Subclass these to create an FSM that fits your tutorial/challenge.
+
+.. automodule:: tfw
+
+.. autoclass:: FSMBase
+    :members:
+
+.. autoclass:: LinearFSM
+    :members:

docs/source/foundations/tfwserver.rst

@@ -0,0 +1,7 @@
+TFWServer
+---------
+
+.. automodule:: tfw.networking
+
+.. autoclass:: TFWServer
+    :members:

docs/source/index.rst

@@ -0,0 +1,45 @@
+.. baseimage-tutorial-framework documentation master file, created by
+   sphinx-quickstart on Fri Jun  1 14:29:07 2018.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to baseimage-tutorial-framework's documentation!
+========================================================
+
+Foundations
+-----------
+
+This part covers the soil which the framework is based on and stuff you will need to develop your own challenges.
+
+.. toctree::
+    :glob:
+
+    foundations/*
+
+Networking
+----------
+
+You can use these to send messages to the frontend or the event handlers through TFWServer.
+
+.. toctree::
+    :glob:
+
+    networking/*
+
+Components
+----------
+
+These are pre-written components for you to use, such as our IDE, terminal or console.
+
+.. toctree::
+    :glob:
+
+    components/*
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/source/networking/networking.rst

@@ -0,0 +1,17 @@
+Networking
+----------
+
+.. automodule:: tfw.networking
+
+.. autoclass:: TFWServerConnector
+    :members:
+
+.. automodule:: tfw.networking.event_handlers
+
+.. autoclass:: ServerUplinkConnector
+    :members:
+
+.. automodule:: tfw.networking
+
+.. autoclass:: MessageSender
+    :members:

lib/envvars.py

@@ -1,12 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-from collections import namedtuple
-from os import environ
-
-
-def prefixed_envvars_to_namedtuple(prefix: str, tuple_name: str):
-    envvars = {envvar.replace(prefix, '', 1): environ.get(envvar)
-               for envvar in environ.keys()
-               if envvar.startswith(prefix)}
-    return namedtuple(tuple_name, envvars)(**envvars)

lib/envvars/__init__.py

@@ -0,0 +1,25 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from collections import namedtuple
+from os import environ
+
+from tfw.decorators import lazy_property
+
+
+class LazyEnvironment:
+    def __init__(self, prefix, tuple_name):
+        self._prefix = prefix
+        self._tuple_name = tuple_name
+
+    @lazy_property
+    def environment(self):
+        return self.prefixed_envvars_to_namedtuple()
+
+    def prefixed_envvars_to_namedtuple(self):
+        envvars = {
+            envvar.replace(self._prefix, '', 1): environ.get(envvar)
+            for envvar in environ.keys()
+            if envvar.startswith(self._prefix)
+        }
+        return namedtuple(self._tuple_name, envvars)(**envvars)

lib/tao/config/envvars.py

@@ -1,6 +1,6 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from envvars import prefixed_envvars_to_namedtuple
+from envvars import LazyEnvironment
 
-TAOENV = prefixed_envvars_to_namedtuple('AVATAO_', 'taoenvtuple')
+TAOENV = LazyEnvironment('AVATAO_', 'taoenvtuple').environment

lib/tfw/__init__.py

@@ -1,6 +1,7 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from .event_handler_base import EventHandlerBase, TriggeredEventHandler
+from .event_handler_base import EventHandlerBase, FSMAwareEventHandler, BroadcastingEventHandler
 from .fsm_base import FSMBase
 from .linear_fsm import LinearFSM
+from .yaml_fsm import YamlFSM

lib/tfw/components/__init__.py

@@ -7,3 +7,8 @@ from .terminal_event_handler import TerminalEventHandler
 from .ide_event_handler import IdeEventHandler
 from .history_monitor import HistoryMonitor, BashMonitor, GDBMonitor
 from .terminal_commands import TerminalCommands
+from .log_monitoring_event_handler import LogMonitoringEventHandler
+from .fsm_managing_event_handler import FSMManagingEventHandler
+from .snapshot_provider import SnapshotProvider
+from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler, PipeIOServer
+from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler

lib/tfw/components/directory_monitor.py

@@ -15,10 +15,11 @@ LOG = logging.getLogger(__name__)
 
 
 class DirectoryMonitor(ObserverMixin):
-    def __init__(self, directory):
-        ObserverMixin.__init__(self)
-        self.eventhandler = IdeReloadWatchdogEventHandler()
-        self.observer.schedule(self.eventhandler, directory, recursive=True)
+    def __init__(self, ide_key, directories):
+        self.eventhandler = IdeReloadWatchdogEventHandler(ide_key)
+        for directory in directories:
+            self.observer.schedule(self.eventhandler, directory, recursive=True)
+
         self.pause, self.resume = self.eventhandler.pause, self.eventhandler.resume
 
     @property
@@ -43,8 +44,9 @@ class DirectoryMonitor(ObserverMixin):
 
 
 class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler):
-    def __init__(self):
+    def __init__(self, ide_key):
         super().__init__()
+        self.ide_key = ide_key
         self.uplink = ServerUplinkConnector()
         self._paused = False
         self.ignore = 0
@@ -63,8 +65,10 @@ class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler):
             self.ignore = self.ignore - 1
             return
         LOG.debug(event)
-        self.uplink.send({'key': 'ide',
-                          'data': {'command': 'reload'}})
+        self.uplink.send({
+            'key': self.ide_key,
+            'data': {'command': 'reload'}
+        })
 
 
 def with_monitor_paused(fun):

lib/tfw/components/directory_monitoring_event_handler.py

@@ -15,12 +15,19 @@ class DirectoryMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin):
     def __init__(self, key, directory):
         super().__init__(key)
         self._directory = directory
-        MonitorManagerMixin.__init__(self, DirectoryMonitor, self._directory)
+        MonitorManagerMixin.__init__(
+            self,
+            DirectoryMonitor,
+            key,
+            self._directory
+        )
 
-        self.commands = {'pause':     self.pause,
-                         'resume':    self.resume,
-                         'ignore':    self.ignore,
-                         'selectdir': self.selectdir}
+        self.commands = {
+            'pause':     self.pause,
+            'resume':    self.resume,
+            'ignore':    self.ignore,
+            'selectdir': self.selectdir
+        }
 
     @property
     def directory(self):

lib/tfw/components/fsm_managing_event_handler.py

@@ -0,0 +1,74 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from tfw import EventHandlerBase
+from tfw.crypto import KeyManager, sign_message, verify_message
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class FSMManagingEventHandler(EventHandlerBase):
+    def __init__(self, key, fsm_type, require_signature=False):
+        super().__init__(key)
+        self.fsm = fsm_type()
+        self._fsm_updater = FSMUpdater(self.fsm)
+        self.auth_key = KeyManager().auth_key
+        self._require_signature = require_signature
+
+        self.command_handlers = {
+            'trigger': self.handle_trigger,
+            'update':  self.handle_update
+        }
+
+    def handle_event(self, message):
+        try:
+            message = self.command_handlers[message['data']['command']](message)
+            if message:
+                fsm_update_message = self._fsm_updater.generate_fsm_update()
+                sign_message(self.auth_key, message)
+                sign_message(self.auth_key, fsm_update_message)
+                self.server_connector.broadcast(fsm_update_message)
+            return message
+        except KeyError:
+            LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
+
+    def handle_trigger(self, message):
+        trigger = message['data']['value']
+        if self._require_signature:
+            if not verify_message(self.auth_key, message):
+                LOG.error('Ignoring unsigned trigger command: %s', message)
+                return None
+        if self.fsm.step(trigger):
+            return message
+        return None
+
+    def handle_update(self, message):
+        # pylint: disable=no-self-use
+        return message
+
+
+class FSMUpdater:
+    def __init__(self, fsm):
+        self.fsm = fsm
+
+    def generate_fsm_update(self):
+        return {
+            'key': 'fsm_update',
+            'data': self.get_fsm_state_and_transitions()
+        }
+
+    def get_fsm_state_and_transitions(self):
+        state = self.fsm.state
+        valid_transitions = [
+            {'trigger': trigger}
+            for trigger in self.fsm.get_triggers(self.fsm.state)
+        ]
+        last_trigger = self.fsm.trigger_history[-1] if self.fsm.trigger_history else None
+        in_accepted_state = state in self.fsm.accepted_states
+        return {
+            'current_state': state,
+            'valid_transitions': valid_transitions,
+            'last_trigger': last_trigger,
+            'in_accepted_state': in_accepted_state
+        }

lib/tfw/components/history_monitor.py

@@ -36,15 +36,17 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC):
     See examples below.
     """
     def __init__(self, histfile):
-        CallbackMixin.__init__(self)
-        ObserverMixin.__init__(self)
         self.histfile = histfile
         self._history = []
         self._last_length = len(self._history)
-        self.observer.schedule(CallbackEventHandler([self.histfile],
-                                                    self._fetch_history,
-                                                    self._invoke_callbacks),
-                               dirname(self.histfile))
+        self.observer.schedule(
+            CallbackEventHandler(
+                [self.histfile],
+                self._fetch_history,
+                self._invoke_callbacks
+            ),
+            dirname(self.histfile)
+        )
 
     @property
     def history(self):
@@ -55,7 +57,10 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC):
         with open(self.histfile, 'r') as ifile:
             pattern = compileregex(self.command_pattern)
             data = ifile.read()
-            self._history = [self.sanitize_command(command) for command in findall(pattern, data)]
+            self._history = [
+                self.sanitize_command(command)
+                for command in findall(pattern, data)
+            ]
 
     @property
     @abstractmethod
@@ -76,10 +81,10 @@ class BashMonitor(HistoryMonitor):
     HistoryMonitor for monitoring bash CLI sessions.
     This requires the following to be set in bash
     (note that this is done automatically by TFW):
-        PROMPT_COMMAND="history -a"
-        shopt -s cmdhist
-        shopt -s histappend
-        unset HISTCONTROL
+    PROMPT_COMMAND="history -a"
+    shopt -s cmdhist
+    shopt -s histappend
+    unset HISTCONTROL
     """
     @property
     def command_pattern(self):

lib/tfw/components/ide_event_handler.py

@@ -4,7 +4,7 @@
 from os.path import isfile, join, relpath, exists, isdir, realpath
 from glob import glob
 from fnmatch import fnmatchcase
-from collections import Iterable
+from typing import Iterable
 
 from tfw import EventHandlerBase
 from tfw.mixins import MonitorManagerMixin
@@ -41,8 +41,8 @@ class FileManager:  # pylint: disable=too-many-instance-attributes
     def workdir(self, directory):
         if not exists(directory) or not isdir(directory):
             raise EnvironmentError(f'"{directory}" is not a directory!')
-        if not self._is_in_whitelisted_dir(directory):
-            raise EnvironmentError(f'Directory "{directory}" is not in whitelist!')
+        if not self._is_in_allowed_dir(directory):
+            raise EnvironmentError(f'Directory "{directory}" is not allowed!')
         self._workdir = directory
 
     @property
@@ -65,8 +65,13 @@ class FileManager:  # pylint: disable=too-many-instance-attributes
 
     @property
     def files(self):
-        return [self._relpath(file) for file in glob(join(self._workdir, '**/*'), recursive=True)
-                if isfile(file) and self._is_in_whitelisted_dir(file) and not self._is_blacklisted(file)]
+        return [
+            self._relpath(file)
+            for file in glob(join(self._workdir, '**/*'), recursive=True)
+            if isfile(file)
+            and self._is_in_allowed_dir(file)
+            and not self._is_blacklisted(file)
+        ]
 
     @property
     def file_contents(self):
@@ -78,11 +83,17 @@ class FileManager:  # pylint: disable=too-many-instance-attributes
         with open(self._filepath(self.filename), 'w', errors='surrogateescape') as ofile:
             ofile.write(value)
 
-    def _is_in_whitelisted_dir(self, path):
-        return any(realpath(path).startswith(allowed_dir) for allowed_dir in self.allowed_directories)
+    def _is_in_allowed_dir(self, path):
+        return any(
+            realpath(path).startswith(allowed_dir)
+            for allowed_dir in self.allowed_directories
+        )
 
     def _is_blacklisted(self, file):
-        return any(fnmatchcase(file, blacklisted) for blacklisted in self.exclude)
+        return any(
+            fnmatchcase(file, blacklisted)
+            for blacklisted in self.exclude
+        )
 
     def _filepath(self, filename):
         return join(self._workdir, filename)
@@ -92,43 +103,61 @@ class FileManager:  # pylint: disable=too-many-instance-attributes
 
 
 class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
-    # pylint: disable=too-many-arguments
+    # pylint: disable=too-many-arguments,anomalous-backslash-in-string
     """
     Event handler implementing the backend of our browser based IDE.
     By default all files in the directory specified in __init__ are displayed
     on the fontend. Note that this is a stateful component.
 
-    This EventHandler accepts messages that have a data["command"] key specifying
+    When any file in the selected directory changes they are automatically refreshed
+    on the frontend (this is done by listening to inotify events).
+
+    This EventHandler accepts messages that have a data['command'] key specifying
     a command to be executed.
-    The API of each command is documented in their respective handlers.
+
+    The API of each command is documented in their respective handler.
     """
     def __init__(self, key, directory, allowed_directories, selected_file=None, exclude=None):
         """
         :param key: the key this instance should listen to
         :param directory: working directory which the EventHandler should serve files from
-        :param allowed_directories: list of directories that can be switched to using the selectdir command
+        :param allowed_directories: list of directories that can be switched to using selectdir
         :param selected_file: file that is selected by default
-        :param exclude: list of filenames that should not appear between files (for *.o, *.pyc, etc.)
+        :param exclude: list of filenames that should not appear between files (for .o, .pyc, etc.)
         """
         super().__init__(key)
         try:
-            self.filemanager = FileManager(allowed_directories=allowed_directories, working_directory=directory,
-                                           selected_file=selected_file, exclude=exclude)
+            self.filemanager = FileManager(
+                allowed_directories=allowed_directories,
+                working_directory=directory,
+                selected_file=selected_file,
+                exclude=exclude
+            )
         except IndexError:
-            raise EnvironmentError(f'No file(s) in IdeEventHandler working_directory "{directory}"!')
-        MonitorManagerMixin.__init__(self, DirectoryMonitor, self.filemanager.workdir)
+            raise EnvironmentError(
+                f'No file(s) in IdeEventHandler working_directory "{directory}"!'
+            )
 
-        self.commands = {'read':      self.read,
-                         'write':     self.write,
-                         'select':    self.select,
-                         'selectdir': self.select_dir,
-                         'exclude':   self.exclude}
+        MonitorManagerMixin.__init__(
+            self,
+            DirectoryMonitor,
+            self.key,
+            self.filemanager.allowed_directories
+        )
+
+        self.commands = {
+            'read':      self.read,
+            'write':     self.write,
+            'select':    self.select,
+            'selectdir': self.select_dir,
+            'exclude':   self.exclude
+        }
 
     def read(self, data):
         """
         Read the currently selected file.
 
-        :return: message with the contents of the file in data['content']
+        :return dict: message with the contents of the file in data['content']
         """
         try:
             data['content'] = self.filemanager.file_contents
@@ -144,8 +173,9 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
         """
         Overwrites a file with the desired string.
 
-        :param data: TFW message data containing keys:
-               |-string: containing the desired file contents
+        :param data: TFW message data containing key 'content'
+                     (new file content)
+
         """
         self.monitor.ignore = self.monitor.ignore + 1
         try:
@@ -159,8 +189,8 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
         """
         Selects a file from the current directory.
 
-        :param data: TFW message data containing keys:
-               |-filename: name of file to select relative to the current directory
+        :param data: TFW message data containing 'filename'
+                     (name of file to select relative to the current directory)
         """
         try:
             self.filemanager.filename = data['filename']
@@ -172,10 +202,10 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
         """
         Select a new working directory to display files from.
 
-        :param data: TFW message data containing keys:
-               |-directory: absolute path of diretory to select.
-                            must be a path whitelisted in
-                            self.allowed_directories
+        :param data: TFW message data containing 'directory'
+                     (absolute path of diretory to select.
+                     must be a path whitelisted in
+                     self.allowed_directories)
         """
         try:
             self.filemanager.workdir = data['directory']
@@ -193,8 +223,9 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
         """
         Overwrite list of excluded files
 
-        :param data: TFW message data containing keys:
-               |-exclude: list of filename patterns to be excluded, e.g.: ["*.pyc", "*.o"]
+        :param data: TFW message data containing 'exclude'
+                     (list of unix-style filename patterns to be excluded,
+                     e.g.: ["\*.pyc", "\*.o")
         """
         try:
             self.filemanager.exclude = list(data['exclude'])

lib/tfw/components/log_monitor.py

@@ -0,0 +1,56 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+import logging
+from os.path import dirname
+
+from watchdog.events import PatternMatchingEventHandler as PatternMatchingWatchdogEventHandler
+
+from tfw.networking.event_handlers import ServerUplinkConnector
+from tfw.decorators import RateLimiter
+from tfw.mixins import ObserverMixin, SupervisorLogMixin
+
+
+class LogMonitor(ObserverMixin):
+    def __init__(self, process_name, log_tail=0):
+        self.prevent_log_recursion()
+        event_handler = SendLogWatchdogEventHandler(
+            process_name,
+            log_tail=log_tail
+        )
+        self.observer.schedule(
+            event_handler,
+            event_handler.path
+        )
+
+    @staticmethod
+    def prevent_log_recursion():
+        # This is done to prevent inotify event logs triggering themselves (infinite log recursion)
+        logging.getLogger('watchdog.observers.inotify_buffer').propagate = False
+
+
+class SendLogWatchdogEventHandler(PatternMatchingWatchdogEventHandler, SupervisorLogMixin):
+    def __init__(self, process_name, log_tail=0):
+        self.process_name = process_name
+        self.procinfo = self.supervisor.getProcessInfo(self.process_name)
+        super().__init__([
+            self.procinfo['stdout_logfile'],
+            self.procinfo['stderr_logfile']
+        ])
+        self.uplink = ServerUplinkConnector()
+        self.log_tail = log_tail
+
+    @property
+    def path(self):
+        return dirname(self.procinfo['stdout_logfile'])
+
+    @RateLimiter(rate_per_second=5)
+    def on_modified(self, event):
+        self.uplink.send({
+            'key': 'processlog',
+            'data': {
+                'command': 'new_log',
+                'stdout': self.read_stdout(self.process_name, tail=self.log_tail),
+                'stderr': self.read_stderr(self.process_name, tail=self.log_tail)
+            }
+        })

lib/tfw/components/log_monitoring_event_handler.py

@@ -0,0 +1,67 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from tfw import EventHandlerBase
+from tfw.mixins import MonitorManagerMixin
+from tfw.config.logs import logging
+from .log_monitor import LogMonitor
+
+LOG = logging.getLogger(__name__)
+
+
+class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin):
+    """
+    Monitors the output of a supervisor process (stdout, stderr) and
+    sends the results to the frontend.
+
+    Accepts messages that have a data['command'] key specifying
+    a command to be executed.
+
+    The API of each command is documented in their respective handler.
+    """
+    def __init__(self, key, process_name, log_tail=0):
+        super().__init__(key)
+        self.process_name = process_name
+        self.log_tail = log_tail
+        MonitorManagerMixin.__init__(
+            self,
+            LogMonitor,
+            self.process_name,
+            self.log_tail
+        )
+
+        self.command_handlers = {
+            'process_name': self.handle_process_name,
+            'log_tail':     self.handle_log_tail
+        }
+
+    def handle_event(self, message):
+        try:
+            data = message['data']
+            self.command_handlers[data['command']](data)
+            self.reload_monitor()
+        except KeyError:
+            LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
+
+    def handle_process_name(self, data):
+        """
+        Changes the monitored process.
+
+        :param data: TFW message data containing 'value'
+                     (name of the process to monitor)
+        """
+        self.set_monitor_args(data['value'], self.log_tail)
+
+    def handle_log_tail(self, data):
+        """
+        Sets tail length of the log the monitor will send
+        to the frontend (the monitor will send back the last
+        'value' characters of the log).
+
+        :param data: TFW message data containing 'value'
+                     (new tail length)
+        """
+        self.set_monitor_args(self.process_name, data['value'])
+
+    def cleanup(self):
+        self.monitor.stop()

lib/tfw/components/pipe_io_event_handler.py

@@ -0,0 +1,143 @@
+from abc import abstractmethod
+from json import loads, dumps
+from subprocess import run, PIPE, Popen
+from functools import partial
+from os import getpgid, killpg
+from os.path import join
+from signal import SIGTERM
+from secrets import token_urlsafe
+from threading import Thread
+from contextlib import suppress
+
+from tfw import EventHandlerBase
+from tfw.config.logs import logging
+
+from .pipe_io_server import PipeIOServer, terminate_process_on_failure
+
+LOG = logging.getLogger(__name__)
+DEFAULT_PERMISSIONS = 0o600
+
+
+class PipeIOEventHandlerBase(EventHandlerBase):
+    def __init__(self, key, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS):
+        super().__init__(key)
+        self.pipe_io = CallbackPipeIOServer(
+            in_pipe_path,
+            out_pipe_path,
+            self.handle_pipe_event,
+            permissions
+        )
+        self.pipe_io.start()
+
+    @abstractmethod
+    def handle_pipe_event(self, message_bytes):
+        raise NotImplementedError()
+
+    def cleanup(self):
+        self.pipe_io.stop()
+
+
+class CallbackPipeIOServer(PipeIOServer):
+    def __init__(self, in_pipe_path, out_pipe_path, callback, permissions):
+        super().__init__(in_pipe_path, out_pipe_path, permissions)
+        self.callback = callback
+
+    def handle_message(self, message):
+        try:
+            self.callback(message)
+        except:  # pylint: disable=bare-except
+            LOG.exception('Failed to handle message %s from pipe %s!', message, self.in_pipe)
+
+
+class PipeIOEventHandler(PipeIOEventHandlerBase):
+    def handle_event(self, message):
+        json_bytes = dumps(message).encode()
+        self.pipe_io.send_message(json_bytes)
+
+    def handle_pipe_event(self, message_bytes):
+        json = loads(message_bytes)
+        self.server_connector.send(json)
+
+
+class TransformerPipeIOEventHandler(PipeIOEventHandlerBase):
+    # pylint: disable=too-many-arguments
+    def __init__(
+            self, key, in_pipe_path, out_pipe_path,
+            transform_in_cmd, transform_out_cmd,
+            permissions=DEFAULT_PERMISSIONS
+    ):
+        self._transform_in = partial(self._transform_message, transform_in_cmd)
+        self._transform_out = partial(self._transform_message, transform_out_cmd)
+        super().__init__(key, in_pipe_path, out_pipe_path, permissions)
+
+    @staticmethod
+    def _transform_message(transform_cmd, message):
+        proc = run(
+            transform_cmd,
+            input=message,
+            stdout=PIPE,
+            stderr=PIPE,
+            shell=True
+        )
+        if proc.returncode == 0:
+            return proc.stdout
+        raise ValueError(f'Transforming message {message} failed!')
+
+    def handle_event(self, message):
+        json_bytes = dumps(message).encode()
+        transformed_bytes = self._transform_out(json_bytes)
+        if transformed_bytes:
+            self.pipe_io.send_message(transformed_bytes)
+
+    def handle_pipe_event(self, message_bytes):
+        transformed_bytes = self._transform_in(message_bytes)
+        if transformed_bytes:
+            json_message = loads(transformed_bytes)
+            self.server_connector.send(json_message)
+
+
+class CommandEventHandler(PipeIOEventHandler):
+    def __init__(self, key, command, permissions=DEFAULT_PERMISSIONS):
+        super().__init__(
+            key,
+            self._generate_tempfilename(),
+            self._generate_tempfilename(),
+            permissions
+        )
+
+        self._proc_stdin = open(self.pipe_io.out_pipe, 'rb')
+        self._proc_stdout = open(self.pipe_io.in_pipe, 'wb')
+        self._proc = Popen(
+            command, shell=True, executable='/bin/bash',
+            stdin=self._proc_stdin, stdout=self._proc_stdout, stderr=PIPE,
+            start_new_session=True
+        )
+        self._monitor_proc_thread = self._start_monitor_proc()
+
+    def _generate_tempfilename(self):
+        # pylint: disable=no-self-use
+        random_filename = partial(token_urlsafe, 10)
+        return join('/tmp', f'{type(self).__name__}.{random_filename()}')
+
+    def _start_monitor_proc(self):
+        thread = Thread(target=self._monitor_proc, daemon=True)
+        thread.start()
+        return thread
+
+    @terminate_process_on_failure
+    def _monitor_proc(self):
+        return_code = self._proc.wait()
+        if return_code == -int(SIGTERM):
+            # supervisord asked the program to terminate, this is fine
+            return
+        if return_code != 0:
+            _, stderr = self._proc.communicate()
+            raise RuntimeError(f'Subprocess failed ({return_code})! Stderr:\n{stderr.decode()}')
+
+    def cleanup(self):
+        with suppress(ProcessLookupError):
+            process_group_id = getpgid(self._proc.pid)
+            killpg(process_group_id, SIGTERM)
+            self._proc_stdin.close()
+            self._proc_stdout.close()
+        super().cleanup()

lib/tfw/components/pipe_io_server/__init__.py

@@ -0,0 +1,2 @@
+from .pipe_io_server import PipeIOServer
+from .terminate_process_on_failure import terminate_process_on_failure

lib/tfw/components/pipe_io_server/deque.py

@@ -0,0 +1,27 @@
+from collections import deque
+from threading import Lock, Condition
+
+
+class Deque:
+    def __init__(self):
+        self._queue = deque()
+
+        self._mutex = Lock()
+        self._not_empty = Condition(self._mutex)
+
+    def pop(self):
+        with self._mutex:
+            while not self._queue:
+                self._not_empty.wait()
+            return self._queue.pop()
+
+    def push(self, item):
+        self._push(item, self._queue.appendleft)
+
+    def push_front(self, item):
+        self._push(item, self._queue.append)
+
+    def _push(self, item, put_method):
+        with self._mutex:
+            put_method(item)
+            self._not_empty.notify()

lib/tfw/components/pipe_io_server/pipe.py

@@ -0,0 +1,16 @@
+from os import mkfifo, remove, chmod
+from os.path import exists
+
+
+class Pipe:
+    def __init__(self, path):
+        self.path = path
+
+    def recreate(self, permissions):
+        self.remove()
+        mkfifo(self.path)
+        chmod(self.path, permissions)  # use chmod to ignore umask
+
+    def remove(self):
+        if exists(self.path):
+            remove(self.path)

lib/tfw/components/pipe_io_server/pipe_io_server.py

@@ -0,0 +1,73 @@
+from abc import ABC, abstractmethod
+from threading import Thread, Event
+from typing import Callable
+
+from .pipe_reader_thread import PipeReaderThread
+from .pipe_writer_thread import PipeWriterThread
+from .pipe import Pipe
+from .terminate_process_on_failure import terminate_process_on_failure
+
+
+class PipeIOServer(ABC, Thread):
+    def __init__(self, in_pipe=None, out_pipe=None, permissions=0o600):
+        super().__init__(daemon=True)
+        self._in_pipe, self._out_pipe = in_pipe, out_pipe
+        self._create_pipes(permissions)
+        self._stop_event = Event()
+        self._reader_thread, self._writer_thread = self._create_io_threads()
+        self._io_threads = (self._reader_thread, self._writer_thread)
+        self._on_stop = lambda: None
+
+    def _create_pipes(self, permissions):
+        Pipe(self.in_pipe).recreate(permissions)
+        Pipe(self.out_pipe).recreate(permissions)
+
+    @property
+    def in_pipe(self):
+        return self._in_pipe
+
+    @property
+    def out_pipe(self):
+        return self._out_pipe
+
+    def _create_io_threads(self):
+        reader_thread = PipeReaderThread(self.in_pipe, self._stop_event, self.handle_message)
+        writer_thread = PipeWriterThread(self.out_pipe, self._stop_event)
+        return reader_thread, writer_thread
+
+    @abstractmethod
+    def handle_message(self, message):
+        raise NotImplementedError()
+
+    def send_message(self, message):
+        self._writer_thread.write(message)
+
+    @terminate_process_on_failure
+    def run(self):
+        for thread in self._io_threads:
+            thread.start()
+        self._stop_event.wait()
+        self._stop_threads()
+
+    def stop(self):
+        self._stop_event.set()
+        if self.is_alive():
+            self.join()
+
+    def _stop_threads(self):
+        for thread in self._io_threads:
+            if thread.is_alive():
+                thread.stop()
+        Pipe(self.in_pipe).remove()
+        Pipe(self.out_pipe).remove()
+        self._on_stop()
+
+    def _set_on_stop(self, value):
+        if not isinstance(value, Callable):
+            raise ValueError("Supplied object is not callable!")
+        self._on_stop = value
+
+    on_stop = property(fset=_set_on_stop)
+
+    def wait(self):
+        self._stop_event.wait()

lib/tfw/components/pipe_io_server/pipe_reader_thread.py

@@ -0,0 +1,44 @@
+from contextlib import suppress
+from os import open as osopen
+from os import write, close, O_WRONLY, O_NONBLOCK
+from threading import Thread
+
+from .terminate_process_on_failure import terminate_process_on_failure
+
+
+class PipeReaderThread(Thread):
+    eof = b''
+    stop_sequence = b'stop_reading\n'
+
+    def __init__(self, pipe_path, stop_event, message_handler):
+        super().__init__(daemon=True)
+        self._message_handler = message_handler
+        self._pipe_path = pipe_path
+        self._stop_event = stop_event
+
+    @terminate_process_on_failure
+    def run(self):
+        with self._open() as pipe:
+            while True:
+                message = pipe.readline()
+                if message == self.stop_sequence:
+                    self._stop_event.set()
+                    break
+                if message == self.eof:
+                    self._open().close()
+                    continue
+                self._message_handler(message[:-1])
+
+    def _open(self):
+        return open(self._pipe_path, 'rb')
+
+    def stop(self):
+        while self.is_alive():
+            self._unblock()
+        self.join()
+
+    def _unblock(self):
+        with suppress(OSError):
+            fd = osopen(self._pipe_path, O_WRONLY | O_NONBLOCK)
+            write(fd, self.stop_sequence)
+            close(fd)

lib/tfw/components/pipe_io_server/pipe_writer_thread.py

@@ -0,0 +1,50 @@
+from contextlib import suppress
+from os import O_NONBLOCK, O_RDONLY, close
+from os import open as osopen
+from threading import Thread
+
+from .terminate_process_on_failure import terminate_process_on_failure
+from .deque import Deque
+
+
+class PipeWriterThread(Thread):
+    def __init__(self, pipe_path, stop_event):
+        super().__init__(daemon=True)
+        self._pipe_path = pipe_path
+        self._stop_event = stop_event
+        self._write_queue = Deque()
+
+    def write(self, message):
+        self._write_queue.push(message)
+
+    @terminate_process_on_failure
+    def run(self):
+        with self._open() as pipe:
+            while True:
+                message = self._write_queue.pop()
+                if message is None:
+                    self._stop_event.set()
+                    break
+                try:
+                    pipe.write(message + b'\n')
+                    pipe.flush()
+                except BrokenPipeError:
+                    try:  # pipe was reopened, close() flushed the message
+                        pipe.close()
+                    except BrokenPipeError:  # close() discarded the message
+                        self._write_queue.push_front(message)
+                    pipe = self._open()
+
+    def _open(self):
+        return open(self._pipe_path, 'wb')
+
+    def stop(self):
+        while self.is_alive():
+            self._unblock()
+        self.join()
+
+    def _unblock(self):
+        with suppress(OSError):
+            fd = osopen(self._pipe_path, O_RDONLY | O_NONBLOCK)
+            self._write_queue.push_front(None)
+            close(fd)

lib/tfw/components/pipe_io_server/terminate_process_on_failure.py

@@ -0,0 +1,15 @@
+from functools import wraps
+from os import kill, getpid
+from signal import SIGTERM
+from traceback import print_exc
+
+
+def terminate_process_on_failure(fun):
+    @wraps(fun)
+    def wrapper(*args, **kwargs):
+        try:
+            return fun(*args, **kwargs)
+        except:  # pylint: disable=bare-except
+            print_exc()
+            kill(getpid(), SIGTERM)
+    return wrapper

lib/tfw/components/process_managing_event_handler.py

@@ -4,19 +4,20 @@
 from xmlrpc.client import Fault as SupervisorFault
 
 from tfw import EventHandlerBase
-from tfw.mixins import SupervisorMixin
+from tfw.mixins import SupervisorMixin, SupervisorLogMixin
 from tfw.config.logs import logging
 from .directory_monitor import with_monitor_paused
 
 LOG = logging.getLogger(__name__)
 
 
-class ProcessManager(SupervisorMixin):
+class ProcessManager(SupervisorMixin, SupervisorLogMixin):
     def __init__(self):
-        self.commands = {'start': self.start_process,
-                         'stop': self.stop_process,
-                         'restart': self.restart_process,
-                         'readlog': self.read_log}
+        self.commands = {
+            'start':   self.start_process,
+            'stop':    self.stop_process,
+            'restart': self.restart_process
+        }
 
     def __call__(self, command, process_name):
         return self.commands[command](process_name)
@@ -26,30 +27,38 @@ class ProcessManagingEventHandler(EventHandlerBase):
     """
     Event handler that can manage processes managed by supervisor.
 
-    This EventHandler accepts messages that have a data["command"] key specifying
+    This EventHandler accepts messages that have a data['command'] key specifying
     a command to be executed.
-    Every message must contain a data["process_name"] field with the name of the
+    Every message must contain a data['process_name'] field with the name of the
     process to manage. This is the name specified in supervisor config files like so:
     [program:someprogram]
 
     Commands available: start, stop, restart, readlog
     (the names are as self-documenting as it gets)
     """
-    def __init__(self, key, dirmonitor=None):
+    def __init__(self, key, dirmonitor=None, log_tail=0):
         super().__init__(key)
-        self.key = key
         self.monitor = dirmonitor
         self.processmanager = ProcessManager()
+        self.log_tail = log_tail
 
     @with_monitor_paused
     def handle_event(self, message):
         try:
             data = message['data']
-            self.processmanager(data['command'], data['process_name'])
+            try:
+                self.processmanager(data['command'], data['process_name'])
+            except SupervisorFault as fault:
+                message['data']['error'] = fault.faultString
+            finally:
+                message['data']['stdout'] = self.processmanager.read_stdout(
+                    data['process_name'],
+                    self.log_tail
+                )
+                message['data']['stderr'] = self.processmanager.read_stderr(
+                    data['process_name'],
+                    self.log_tail
+                )
             return message
         except KeyError:
             LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
-        except SupervisorFault as fault:
-            message['data']['error'] = fault.faultString
-            message['data']['log'] = self.processmanager.read_log(message['data']['process_name'])
-            return message

lib/tfw/components/snapshot_provider.py

@@ -0,0 +1,175 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+import re
+from subprocess import run, CalledProcessError
+from getpass import getuser
+from os.path import isdir
+from datetime import datetime
+from uuid import uuid4
+
+
+class SnapshotProvider:
+    def __init__(self, directory, git_dir):
+        self._classname = self.__class__.__name__
+        author = f'{getuser()} via TFW {self._classname}'
+        self.gitenv = {
+            'GIT_DIR': git_dir,
+            'GIT_WORK_TREE': directory,
+            'GIT_AUTHOR_NAME': author,
+            'GIT_AUTHOR_EMAIL': '',
+            'GIT_COMMITTER_NAME': author,
+            'GIT_COMMITTER_EMAIL': '',
+            'GIT_PAGER': 'cat'
+        }
+
+        self._init_repo()
+        self.__last_valid_branch = self._branch
+
+    def _init_repo(self):
+        self._check_environment()
+
+        if not self._repo_is_initialized:
+            self._run(('git', 'init'))
+
+        if self._number_of_commits == 0:
+            try:
+                self._snapshot()
+            except CalledProcessError:
+                raise EnvironmentError(f'{self._classname} cannot init on empty directories!')
+
+        self._check_head_not_detached()
+
+    def _check_environment(self):
+        if not isdir(self.gitenv['GIT_DIR']) or not isdir(self.gitenv['GIT_WORK_TREE']):
+            raise EnvironmentError(f'{self._classname}: "directory" and "git_dir" must exist!')
+
+    @property
+    def _repo_is_initialized(self):
+        return self._run(
+            ('git', 'status'),
+            check=False
+        ).returncode == 0
+
+    @property
+    def _number_of_commits(self):
+        return int(
+            self._get_stdout((
+                'git', 'rev-list',
+                '--all',
+                '--count'
+            ))
+        )
+
+    def _snapshot(self):
+        self._run((
+            'git', 'add',
+            '-A'
+        ))
+        self._run((
+            'git', 'commit',
+            '-m', 'Snapshot'
+        ))
+
+    def _check_head_not_detached(self):
+        if self._head_detached:
+            raise EnvironmentError(f'{self._classname} cannot init from detached HEAD state!')
+
+    @property
+    def _head_detached(self):
+        return self._branch == 'HEAD'
+
+    @property
+    def _branch(self):
+        return self._get_stdout((
+            'git', 'rev-parse',
+            '--abbrev-ref', 'HEAD'
+        ))
+
+    def _get_stdout(self, *args, **kwargs):
+        kwargs['capture_output'] = True
+        stdout_bytes = self._run(*args, **kwargs).stdout
+        return stdout_bytes.decode().rstrip('\n')
+
+    def _run(self, *args, **kwargs):
+        if 'check' not in kwargs:
+            kwargs['check'] = True
+        if 'env' not in kwargs:
+            kwargs['env'] = self.gitenv
+        return run(*args, **kwargs)
+
+    def take_snapshot(self):
+        if self._head_detached:
+            self._checkout_new_branch_from_head()
+        self._snapshot()
+
+    def _checkout_new_branch_from_head(self):
+        branch_name = uuid4()
+        self._run((
+            'git', 'branch',
+            branch_name
+        ))
+        self._checkout(branch_name)
+
+    def _checkout(self, what):
+        self._run((
+            'git', 'checkout',
+            what
+        ))
+
+    def restore_snapshot(self, date):
+        commit = self._get_commit_from_timestamp(date)
+        self._checkout(commit)
+
+    def _get_commit_from_timestamp(self, date):
+        return self._get_stdout((
+            'git', 'rev-list',
+            '--date=iso',
+            '-n', '1',
+            f'--before="{date.isoformat()}"',
+            self._last_valid_branch
+        ))
+
+    @property
+    def _last_valid_branch(self):
+        if not self._head_detached:
+            self.__last_valid_branch = self._branch
+        return self.__last_valid_branch
+
+    @property
+    def all_timelines(self):
+        return self._branches
+
+    @property
+    def _branches(self):
+        git_branch_output = self._get_stdout(('git', 'branch'))
+        regex_pattern = re.compile(r'(?:[^\S\n]|[*])')  # matches '*' and non-newline whitespace chars
+        return re.sub(regex_pattern, '', git_branch_output).splitlines()
+
+    @property
+    def timeline(self):
+        return self._last_valid_branch
+
+    @timeline.setter
+    def timeline(self, value):
+        self._checkout(value)
+
+    @property
+    def snapshots(self):
+        return self._pretty_log_branch()
+
+    def _pretty_log_branch(self):
+        git_log_output = self._get_stdout((
+            'git', 'log',
+            '--pretty=%H@%aI'
+        ))
+
+        commits = []
+        for line in git_log_output.splitlines():
+            commit_hash, timestamp = line.split('@')
+            commits.append({
+                'hash': commit_hash,
+                'timestamp': datetime.fromisoformat(timestamp)
+            })
+
+        return commits

lib/tfw/components/terminado_mini_server.py

@@ -14,15 +14,15 @@ LOG = logging.getLogger(__name__)
 class TerminadoMiniServer:
     def __init__(self, url, port, workdir, shellcmd):
         self.port = port
-        self._term_manager = SingleTermManager(shell_command=shellcmd,
-                                               term_settings={'cwd': workdir})
-        self.application = Application(
-            [(
-                url,
-                TerminadoMiniServer.ResetterTermSocket,
-                {'term_manager': self._term_manager}
-            )]
+        self._term_manager = SingleTermManager(
+            shell_command=shellcmd,
+            term_settings={'cwd': workdir}
         )
+        self.application = Application([(
+            url,
+            TerminadoMiniServer.ResetterTermSocket,
+            {'term_manager': self._term_manager}
+        )])
 
     @property
     def term_manager(self):
@@ -46,5 +46,10 @@ class TerminadoMiniServer:
 
 if __name__ == '__main__':
     LOG.info('Terminado Mini Server listening on %s', TFWENV.TERMINADO_PORT)
-    TerminadoMiniServer('/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, ['bash']).listen()
+    TerminadoMiniServer(
+        '/terminal',
+        TFWENV.TERMINADO_PORT,
+        TFWENV.TERMINADO_WD,
+        ['bash']
+    ).listen()
     IOLoop.instance().start()

lib/tfw/components/terminal_commands.py

@@ -3,6 +3,7 @@
 
 from abc import ABC
 from re import match
+from shlex import split
 
 from tfw.config.logs import logging
 
@@ -10,6 +11,7 @@ LOG = logging.getLogger(__name__)
 
 
 class TerminalCommands(ABC):
+    # pylint: disable=anomalous-backslash-in-string
     """
     A class you can use to define hooks for terminal commands. This means that you can
     have python code executed when the user enters a specific command to the terminal on
@@ -18,13 +20,13 @@ class TerminalCommands(ABC):
     To receive events you need to subscribe TerminalCommand.callback to a HistoryMonitor
     instance.
 
-    Inherit from this class and define methods which start with "command_". When the user
+    Inherit from this class and define methods which start with "command\_". When the user
     executes the command specified after the underscore, your method will be invoked. All
-    such commands must expect the parameter *args which will contain the arguments of the
+    such commands must expect the parameter \*args which will contain the arguments of the
     command.
 
     For example to define a method that runs when someone starts vim in the terminal
-    you have to define a method like: "def command_vim(self, *args)"
+    you have to define a method like: "def command_vim(self, \*args)"
 
     You can also use this class to create new commands similarly.
     """
@@ -35,8 +37,12 @@ class TerminalCommands(ABC):
             self._setup_bashrc_aliases(bashrc)
 
     def _build_command_to_implementation_dict(self):
-        return {self._parse_command_name(fun): getattr(self, fun) for fun in dir(self)
-                if callable(getattr(self, fun)) and self._is_command_implementation(fun)}
+        return {
+            self._parse_command_name(fun): getattr(self, fun)
+            for fun in dir(self)
+            if callable(getattr(self, fun))
+            and self._is_command_implementation(fun)
+        }
 
     def _setup_bashrc_aliases(self, bashrc):
         with open(bashrc, 'a') as ofile:
@@ -57,10 +63,10 @@ class TerminalCommands(ABC):
         return match(self._command_method_regex, string)
 
     def callback(self, history):
-        parts = history[-1].split()
+        parts = split(history[-1])
         command = parts[0]
         if command in self.command_implemetations.keys():
             try:
                 self.command_implemetations[command](*parts[1:])
-            except IndexError:
-                LOG.debug('Command "%s" failed!', command)
+            except Exception: # pylint: disable=broad-except
+                LOG.exception('Command "%s" failed:', command)

lib/tfw/components/terminal_event_handler.py

@@ -16,9 +16,9 @@ class TerminalEventHandler(EventHandlerBase):
     sessions to connect to. You need to instanciate this in order for frontend
     terminals to work.
 
-    This EventHandler accepts messages that have a data["command"] key specifying
+    This EventHandler accepts messages that have a data['command'] key specifying
     a command to be executed.
-    The API of each command is documented in their respective handlers.
+    The API of each command is documented in their respective handler.
     """
     def __init__(self, key, monitor):
         """
@@ -29,9 +29,19 @@ class TerminalEventHandler(EventHandlerBase):
         self.working_directory = TFWENV.TERMINADO_DIR
         self._historymonitor = monitor
         bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash']
-        self.terminado_server = TerminadoMiniServer('/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, bash_as_user_cmd)
-        self.commands = {'write': self.write,
-                         'read': self.read}
+
+        self.terminado_server = TerminadoMiniServer(
+            '/terminal',
+            TFWENV.TERMINADO_PORT,
+            TFWENV.TERMINADO_WD,
+            bash_as_user_cmd
+        )
+
+        self.commands = {
+            'write': self.write,
+            'read': self.read
+        }
+
         if self._historymonitor:
             self._historymonitor.watch()
         self.terminado_server.listen()
@@ -54,18 +64,19 @@ class TerminalEventHandler(EventHandlerBase):
         Writes a string to the terminal session (on the pty level).
         Useful for pre-typing and executing commands for the user.
 
-        :param data: TFW message data containing keys:
-               |-shellcmd: command to be written to the pty
+        :param data: TFW message data containing 'value'
+                     (command to be written to the pty)
         """
-        self.terminado_server.pty.write(data['shellcmd'])
+        self.terminado_server.pty.write(data['value'])
+        return data
 
     def read(self, data):
         """
         Reads the history of commands executed.
 
-        :param data: TFW message data containing keys:
-               |-count: the number of history elements to return
-        :return: message with list of commands in data['history']
+        :param data: TFW message data containing 'count'
+                     (the number of history elements to return)
+        :return dict: message with list of commands in data['history']
         """
         data['count'] = int(data.get('count', 1))
         if self.historymonitor:

lib/tfw/config/envvars.py

@@ -1,6 +1,6 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from envvars import prefixed_envvars_to_namedtuple
+from envvars import LazyEnvironment
 
-TFWENV = prefixed_envvars_to_namedtuple('TFW_', 'tfwenvtuple')
+TFWENV = LazyEnvironment('TFW_', 'tfwenvtuple').environment

lib/tfw/crypto.py

@@ -0,0 +1,107 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from functools import wraps
+from base64 import b64encode, b64decode
+from copy import deepcopy
+from hashlib import md5
+from os import urandom, chmod
+from os.path import exists
+from stat import S_IRUSR, S_IWUSR, S_IXUSR
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.hashes import SHA256
+from cryptography.hazmat.primitives.hmac import HMAC as _HMAC
+from cryptography.exceptions import InvalidSignature
+
+from tfw.networking import message_bytes
+from tfw.decorators import lazy_property
+from tfw.config import TFWENV
+
+
+def message_checksum(message):
+    return md5(message_bytes(message)).hexdigest()
+
+
+def sign_message(key, message):
+    signature = message_signature(key, message)
+    message['signature'] = b64encode(signature).decode()
+
+
+def message_signature(key, message):
+    return HMAC(key, message_bytes(message)).signature
+
+
+def verify_message(key, message):
+    message = deepcopy(message)
+    try:
+        signature_b64 = message.pop('signature')
+        signature = b64decode(signature_b64)
+        actual_signature = message_signature(key, message)
+        return signature == actual_signature
+    except KeyError:
+        return False
+
+
+class KeyManager:
+    def __init__(self):
+        self.keyfile = TFWENV.AUTH_KEY
+        if not exists(self.keyfile):
+            self._init_auth_key()
+
+    @lazy_property
+    def auth_key(self):
+        with open(self.keyfile, 'rb') as ifile:
+            return ifile.read()
+
+    def _init_auth_key(self):
+        key = self.generate_key()
+        with open(self.keyfile, 'wb') as ofile:
+            ofile.write(key)
+        self._chmod_700_keyfile()
+        return key
+
+    @staticmethod
+    def generate_key():
+        return urandom(32)
+
+    def _chmod_700_keyfile(self):
+        chmod(self.keyfile, S_IRUSR | S_IWUSR | S_IXUSR)
+
+
+class HMAC:
+    def __init__(self, key, message):
+        self.key = key
+        self.message = message
+        self._hmac = _HMAC(
+            key=key,
+            algorithm=SHA256(),
+            backend=default_backend()
+        )
+
+    def _reload_if_finalized(f):
+        # pylint: disable=no-self-argument,not-callable
+        @wraps(f)
+        def wrapped(instance, *args, **kwargs):
+            if getattr(instance, '_finalized', False):
+                instance.__init__(instance.key, instance.message)
+            ret_val = f(instance, *args, **kwargs)
+            setattr(instance, '_finalized', True)
+            return ret_val
+        return wrapped
+
+    @property
+    @_reload_if_finalized
+    def signature(self):
+        self._hmac.update(self.message)
+        signature = self._hmac.finalize()
+        return signature
+
+    @_reload_if_finalized
+    def verify(self, signature):
+        self._hmac.update(self.message)
+        try:
+            self._hmac.verify(signature)
+            return True
+        except InvalidSignature:
+            return False

lib/tfw/decorators/__init__.py

@@ -2,3 +2,4 @@
 # All Rights Reserved. See LICENSE file for details.
 
 from .rate_limiter import RateLimiter
+from .lazy_property import lazy_property

lib/tfw/decorators/lazy_property.py

@@ -0,0 +1,21 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from functools import update_wrapper
+
+
+class lazy_property:
+    """
+    Decorator that replaces a function with the value
+    it calculates on the first call.
+    """
+    def __init__(self, func):
+        self.func = func
+        update_wrapper(self, func)
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self  # avoids potential __new__ TypeError
+        value = self.func(instance)
+        setattr(instance, self.func.__name__, value)
+        return value

lib/tfw/decorators/rate_limiter.py

@@ -1,25 +1,100 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from functools import wraps
+from functools import wraps, partial
 from time import time, sleep
 
+from tfw.decorators.lazy_property import lazy_property
+
 
 class RateLimiter:
+    """
+    Decorator class for rate limiting, blocking.
+
+    When applied to a function this decorator will apply rate limiting
+    if the function is invoked more frequently than rate_per_seconds.
+
+    By default rate limiting means sleeping until the next invocation time
+    as per __init__ parameter rate_per_seconds.
+
+    Note that this decorator BLOCKS THE THREAD it is being executed on,
+    so it is only acceptable for stuff running on a separate thread.
+
+    If this is no good for you please refer to AsyncRateLimiter in this module,
+    which is designed not to block and use the IOLoop it is being called from.
+    """
     def __init__(self, rate_per_second):
+        """
+        :param rate_per_second: max frequency the decorated method should be
+                                invoked with
+        """
         self.min_interval = 1 / float(rate_per_second)
+        self.fun = None
         self.last_call = time()
 
+    def action(self, seconds_to_next_call):
+        if seconds_to_next_call:
+            sleep(seconds_to_next_call)
+        self.fun()
+
     def __call__(self, fun):
         @wraps(fun)
         def wrapper(*args, **kwargs):
-            self._limit_rate()
-            fun(*args, **kwargs)
+            self.fun = partial(fun, *args, **kwargs)
+            limit_seconds = self._limit_rate()
+            self.action(limit_seconds)
         return wrapper
 
     def _limit_rate(self):
-        since_last_call = time() - self.last_call
-        to_next_call = self.min_interval - since_last_call
+        seconds_since_last_call = time() - self.last_call
+        seconds_to_next_call = self.min_interval - seconds_since_last_call
+
+        if seconds_to_next_call > 0:
+            return seconds_to_next_call
         self.last_call = time()
-        if to_next_call > 0:
-            sleep(to_next_call)
+        return 0
+
+
+class AsyncRateLimiter(RateLimiter):
+    """
+    Decorator class for rate limiting, non-blocking.
+
+    The semantics of the rate limiting:
+        - unlike RateLimiter this decorator never blocks, instead it adds an async
+          callback version of the decorated function to the IOLoop
+          (to be executed after the rate limiting has expired).
+        - the timing works similarly to RateLimiter
+    """
+    def __init__(self, rate_per_second, ioloop_factory):
+        """
+        :param rate_per_second: max frequency the decorated method should be
+                                invoked with
+        :param ioloop_factory: callable that should return an instance of the
+                               IOLoop of the application
+        """
+        self._ioloop_factory = ioloop_factory
+        self._ioloop = None
+        self._last_callback = None
+
+        self._make_action_thread_safe()
+        super().__init__(rate_per_second=rate_per_second)
+
+    def _make_action_thread_safe(self):
+        self.action = partial(self.ioloop.add_callback, self.action)
+
+    @lazy_property
+    def ioloop(self):
+        return self._ioloop_factory()
+
+    def action(self, seconds_to_next_call):
+        if self._last_callback:
+            self.ioloop.remove_timeout(self._last_callback)
+
+        self._last_callback = self.ioloop.call_later(
+            seconds_to_next_call,
+            self.fun_with_debounce
+        )
+
+    def fun_with_debounce(self):
+        self.last_call = time()
+        self.fun()

lib/tfw/event_handler_base.py

@@ -2,9 +2,14 @@
 # All Rights Reserved. See LICENSE file for details.
 
 from abc import ABC, abstractmethod
+from inspect import currentframe
+from typing import Iterable
 
-from tfw.networking import deserialize_tfw_msg
 from tfw.networking.event_handlers import ServerConnector
+from tfw.crypto import message_checksum, KeyManager, verify_message
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
 
 
 class EventHandlerBase(ABC):
@@ -16,30 +21,58 @@ class EventHandlerBase(ABC):
     """
     def __init__(self, key):
         self.server_connector = ServerConnector()
-        self.key = key
-        self.subscribe(self.key, 'reset')
+        self.keys = []
+        if isinstance(key, str):
+            self.keys.append(key)
+        elif isinstance(key, Iterable):
+            self.keys = list(key)
+
+        self.subscribe(*self.keys)
         self.server_connector.register_callback(self.event_handler_callback)
 
-    def event_handler_callback(self, msg_parts):
+    @property
+    def key(self):
+        """
+        Returns the oldest key this EventHandler was subscribed to.
+        """
+        return self.keys[0]
+
+    def event_handler_callback(self, message):
         """
         Callback that is invoked when receiving a message.
         Dispatches messages to handler methods and sends
         a response back in case the handler returned something.
         This is subscribed in __init__().
         """
-        message = deserialize_tfw_msg(*msg_parts)
+        if not self.check_key(message):
+            return
+
         response = self.dispatch_handling(message)
         if response:
-            response['key'] = message['key']
             self.server_connector.send(response)
 
+    def check_key(self, message):
+        """
+        Checks whether the message is intended for this
+        EventHandler.
+
+        This is necessary because ZMQ handles PUB - SUB
+        connetions with pattern matching (e.g. someone
+        subscribed to 'fsm' will receive 'fsm_update'
+        messages as well.
+        """
+        if '' in self.keys:
+            return True
+        return message['key'] in self.keys
+
     def dispatch_handling(self, message):
         """
         Used to dispatch messages to their specific handlers.
+
+        :param message: the message received
+        :returns: the message to send back
         """
-        if message['key'] != 'reset':
-            return self.handle_event(message)
-        return self.handle_reset(message)
+        return self.handle_event(message)
 
     @abstractmethod
     def handle_event(self, message):
@@ -47,18 +80,10 @@ class EventHandlerBase(ABC):
         Abstract method that implements the handling of messages.
 
         :param message: the message received
+        :returns: the message to send back
         """
         raise NotImplementedError
 
-    def handle_reset(self, message):
-        # pylint: disable=unused-argument,no-self-use
-        """
-        Usually 'reset' events receive some sort of special treatment.
-
-        :param message: the message received
-        """
-        return None
-
     def subscribe(self, *keys):
         """
         Subscribe this EventHandler to receive events for given keys.
@@ -70,6 +95,7 @@ class EventHandlerBase(ABC):
         """
         for key in keys:
             self.server_connector.subscribe(key)
+            self.keys.append(key)
 
     def unsubscribe(self, *keys):
         """
@@ -79,6 +105,7 @@ class EventHandlerBase(ABC):
         """
         for key in keys:
             self.server_connector.unsubscribe(key)
+            self.keys.remove(key)
 
     def cleanup(self):
         """
@@ -87,18 +114,75 @@ class EventHandlerBase(ABC):
         """
         pass
 
+    @classmethod
+    def get_local_instances(cls):
+        frame = currentframe()
+        if frame is None:
+            raise EnvironmentError('inspect.currentframe() is not supported!')
 
-class TriggeredEventHandler(EventHandlerBase, ABC):
+        locals_values = frame.f_back.f_locals.values()
+        return {
+            instance for instance in locals_values
+            if isinstance(instance, cls)
+        }
+
+
+class FSMAwareEventHandler(EventHandlerBase, ABC):
     # pylint: disable=abstract-method
     """
-    Abstract base class for EventHandlers which are only triggered in case
-    TFWServer has successfully triggered an FSM step defined in __init__.
+    Abstract base class for EventHandlers which automatically
+    keep track of the state of the TFW FSM.
     """
-    def __init__(self, key, trigger):
+    def __init__(self, key):
         super().__init__(key)
-        self.trigger = trigger
+        self.subscribe('fsm_update')
+        self.fsm_state = None
+        self.in_accepted_state = False
+        self._auth_key = KeyManager().auth_key
 
     def dispatch_handling(self, message):
-        if message.get('trigger') == self.trigger:
-            return super().dispatch_handling(message)
-        return None
+        if message['key'] == 'fsm_update':
+            if verify_message(self._auth_key, message):
+                self._handle_fsm_update(message)
+            return None
+        return super().dispatch_handling(message)
+
+    def _handle_fsm_update(self, message):
+        try:
+            new_state = message['data']['current_state']
+            trigger = message['data']['last_trigger']
+            if self.fsm_state != new_state:
+                self.handle_fsm_step(self.fsm_state, new_state, trigger)
+            self.fsm_state = new_state
+            self.in_accepted_state = message['data']['in_accepted_state']
+        except KeyError:
+            LOG.error('Invalid fsm_update message received!')
+
+    def handle_fsm_step(self, from_state, to_state, trigger):
+        """
+        Called in case the TFW FSM has stepped.
+        """
+        pass
+
+
+class BroadcastingEventHandler(EventHandlerBase, ABC):
+    # pylint: disable=abstract-method
+    """
+    Abstract base class for EventHandlers which broadcast responses
+    and intelligently ignore their own broadcasted messages they receive.
+    """
+    def __init__(self, key):
+        super().__init__(key)
+        self.own_message_hashes = []
+
+    def event_handler_callback(self, message):
+        message_hash = message_checksum(message)
+
+        if message_hash in self.own_message_hashes:
+            self.own_message_hashes.remove(message_hash)
+            return
+
+        response = self.dispatch_handling(message)
+        if response:
+            self.own_message_hashes.append(message_checksum(response))
+            self.server_connector.broadcast(response)

lib/tfw/fsm_base.py

@@ -1,36 +1,68 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from typing import List
+from collections import defaultdict
 
-from transitions import Machine
+from transitions import Machine, MachineError
 
 from tfw.mixins import CallbackMixin
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
 
 
-class FSMBase(CallbackMixin):
+class FSMBase(Machine, CallbackMixin):
     """
     A general FSM base class you can inherit from to track user progress.
     See linear_fsm.py for an example use-case.
-    TFW the transitions library for state machines, please refer to their
+    TFW uses the transitions library for state machines, please refer to their
     documentation for more information on creating your own machines:
     https://github.com/pytransitions/transitions
     """
     states, transitions = [], []
 
-    def __init__(self, initial: str = None, accepted_states: List[str] = None):
-        CallbackMixin.__init__(self)
-        self.accepted_states = accepted_states or [self.states[-1]]
-        self.machine = Machine(model=self,
-                               states=self.states,
-                               transitions=self.transitions,
-                               initial=initial or self.states[0],
-                               send_event=True,
-                               ignore_invalid_triggers=True,
-                               after_state_change='execute_callbacks')
+    def __init__(self, initial=None, accepted_states=None):
+        self.accepted_states = accepted_states or [self.states[-1].name]
+        self.trigger_predicates = defaultdict(list)
+        self.trigger_history = []
+
+        Machine.__init__(
+            self,
+            states=self.states,
+            transitions=self.transitions,
+            initial=initial or self.states[0],
+            send_event=True,
+            ignore_invalid_triggers=True,
+            after_state_change='execute_callbacks'
+        )
 
     def execute_callbacks(self, event_data):
         self._execute_callbacks(event_data.kwargs)
 
     def is_solved(self):
-        return self.state in self.accepted_states # pylint: disable=no-member
+        return self.state in self.accepted_states  # pylint: disable=no-member
+
+    def subscribe_predicate(self, trigger, *predicates):
+        self.trigger_predicates[trigger].extend(predicates)
+
+    def unsubscribe_predicate(self, trigger, *predicates):
+        self.trigger_predicates[trigger] = [
+            predicate
+            for predicate in self.trigger_predicates[trigger]
+            not in predicates
+        ]
+
+    def step(self, trigger):
+        predicate_results = (
+            predicate()
+            for predicate in self.trigger_predicates[trigger]
+        )
+
+        if all(predicate_results):
+            try:
+                self.trigger(trigger)
+                self.trigger_history.append(trigger)
+                return True
+            except (AttributeError, MachineError):
+                LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
+        return False

lib/tfw/linear_fsm.py

@@ -1,6 +1,8 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
+from transitions import State
+
 from .fsm_base import FSMBase
 
 
@@ -11,12 +13,19 @@ class LinearFSM(FSMBase):
     a number of steps specified in the constructor. It automatically sets up 2
     actions (triggers) between states as such:
     (0) --  step_1  --> (1) --  step_2  --> (2) --  step_3  --> (3) ... and so on
-        \-step_next-/       \-step_next-/       \-step_next-/
     """
     def __init__(self, number_of_steps):
-        self.states = list(map(str, range(number_of_steps)))
+        self.states = [State(name=str(index)) for index in range(number_of_steps)]
         self.transitions = []
-        for index in self.states[:-1]:
-            self.transitions.append({'trigger': f'step_{int(index)+1}', 'source': index, 'dest': str(int(index)+1)})
-            self.transitions.append({'trigger': 'step_next', 'source': index, 'dest': str(int(index)+1)})
+        for state in self.states[:-1]:
+            self.transitions.append({
+                'trigger': f'step_{int(state.name)+1}',
+                'source': state.name,
+                'dest': str(int(state.name)+1)
+            })
+            self.transitions.append({
+                'trigger': 'step_next',
+                'source': state.name,
+                'dest': str(int(state.name)+1)
+            })
         super(LinearFSM, self).__init__()

lib/tfw/mixins/__init__.py

@@ -1,7 +1,7 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from .supervisor_mixin import SupervisorMixin
+from .supervisor_mixin import SupervisorMixin, SupervisorLogMixin
 from .callback_mixin import CallbackMixin
 from .observer_mixin import ObserverMixin
 from .monitor_manager_mixin import MonitorManagerMixin

lib/tfw/mixins/callback_mixin.py

@@ -3,21 +3,33 @@
 
 from functools import partial
 
+from tfw.decorators import lazy_property
+
 
 class CallbackMixin:
-    def __init__(self):
-        self._callbacks = []
+    @lazy_property
+    def _callbacks(self):
+        # pylint: disable=no-self-use
+        return []
 
     def subscribe_callback(self, callback, *args, **kwargs):
         """
         Subscribe a callable to invoke once an event is triggered.
         :param callback: callable to be executed on events
-        :param *args: arguments passed to callable
-        :param **kwargs: kwargs passed to callable
+        :param args: arguments passed to callable
+        :param kwargs: kwargs passed to callable
         """
         fun = partial(callback, *args, **kwargs)
         self._callbacks.append(fun)
 
+    def subscribe_callbacks(self, *callbacks):
+        """
+        Subscribe a list of callbacks to incoke once an event is triggered.
+        :param callbacks: callbacks to be subscribed
+        """
+        for callback in callbacks:
+            self.subscribe_callback(callback)
+
     def unsubscribe_callback(self, callback):
         self._callbacks.remove(callback)
 

lib/tfw/mixins/monitor_manager_mixin.py

@@ -7,21 +7,24 @@ LOG = logging.getLogger(__name__)
 
 
 class MonitorManagerMixin:
-    def __init__(self, monitor_type, directory):
+    def __init__(self, monitor_type, *monitor_args):
         self._monitor_type = monitor_type
         self._monitor = None
-        self._monitored_directory = directory
+        self.monitor_args = monitor_args
         self.reload_monitor()
 
     @property
     def monitor(self):
         return self._monitor
 
+    def set_monitor_args(self, *monitor_args):
+        self.monitor_args = monitor_args
+
     def reload_monitor(self):
         if self._monitor:
             try:
                 self._monitor.stop()
             except KeyError:
                 LOG.debug('Working directory was removed – ignoring...')
-        self._monitor = self._monitor_type(self._monitored_directory)
+        self._monitor = self._monitor_type(*self.monitor_args)
         self._monitor.watch()   # This runs on a separate thread

lib/tfw/mixins/observer_mixin.py

@@ -3,10 +3,14 @@
 
 from watchdog.observers import Observer
 
+from tfw.decorators import lazy_property
+
 
 class ObserverMixin:
-    def __init__(self):
-        self.observer = Observer()
+    @lazy_property
+    def observer(self):
+        # pylint: disable=no-self-use
+        return Observer()
 
     def watch(self):
         self.observer.start()

lib/tfw/mixins/supervisor_mixin.py

@@ -6,12 +6,18 @@ from xmlrpc.client import Fault as SupervisorFault
 from contextlib import suppress
 from os import remove
 
+from tfw.decorators import lazy_property
 from tfw.config import TFWENV
 
 
-class SupervisorMixin:
-    supervisor = xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor
+class SupervisorBaseMixin:
+    @lazy_property
+    def supervisor(self):
+        # pylint: disable=no-self-use
+        return xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor
 
+
+class SupervisorMixin(SupervisorBaseMixin):
     def stop_process(self, process_name):
         with suppress(SupervisorFault):
             self.supervisor.stopProcess(process_name)
@@ -19,12 +25,20 @@ class SupervisorMixin:
     def start_process(self, process_name):
         self.supervisor.startProcess(process_name)
 
-    def read_log(self, process_name):
-        logs = self.supervisor.readProcessStderrLog(process_name, 0, 0)
-        remove(self.supervisor.getProcessInfo(process_name)['stderr_logfile'])
-        self.supervisor.clearProcessLogs(process_name)
-        return logs
-
     def restart_process(self, process_name):
         self.stop_process(process_name)
         self.start_process(process_name)
+
+
+class SupervisorLogMixin(SupervisorBaseMixin):
+    def read_stdout(self, process_name, tail=0):
+        return self.supervisor.readProcessStdoutLog(process_name, -tail, 0)
+
+    def read_stderr(self, process_name, tail=0):
+        return self.supervisor.readProcessStderrLog(process_name, -tail, 0)
+
+    def clear_logs(self, process_name):
+        for logfile in ('stdout_logfile', 'stderr_logfile'):
+            with suppress(FileNotFoundError):
+                remove(self.supervisor.getProcessInfo(process_name)[logfile])
+        self.supervisor.clearProcessLogs(process_name)

lib/tfw/networking/__init__.py

@@ -1,9 +1,9 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from .serialization import serialize_tfw_msg, deserialize_tfw_msg, validate_message
+from .serialization import serialize_tfw_msg, deserialize_tfw_msg
+from .serialization import with_deserialize_tfw_msg, message_bytes
 from .zmq_connector_base import ZMQConnectorBase
-# from .controller_connector import ControllerConnector # TODO: readd once controller stuff is resolved
 from .message_sender import MessageSender
 from .event_handlers.server_connector import ServerUplinkConnector as TFWServerConnector
 from .server.tfw_server import TFWServer

lib/tfw/networking/controller_connector.py

@@ -1,18 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-import zmq
-from zmq.eventloop.zmqstream import ZMQStream
-
-from tfw.config import TFWENV
-from tfw.networking import ZMQConnectorBase
-
-
-class ControllerConnector(ZMQConnectorBase):
-    def __init__(self, zmq_context=None):
-        super(ControllerConnector, self).__init__(zmq_context)
-        self._zmq_rep_socket = self._zmq_context.socket(zmq.REP)
-        self._zmq_rep_socket.connect(f'tcp://localhost:{TFWENV.CONTROLLER_PORT}')
-        self._zmq_rep_stream = ZMQStream(self._zmq_rep_socket)
-
-        self.register_callback = self._zmq_rep_stream.on_recv_stream

lib/tfw/networking/event_handlers/server_connector.py

@@ -6,9 +6,12 @@ from functools import partial
 import zmq
 from zmq.eventloop.zmqstream import ZMQStream
 
-from tfw.networking import serialize_tfw_msg
+from tfw.networking import serialize_tfw_msg, with_deserialize_tfw_msg
 from tfw.networking import ZMQConnectorBase
 from tfw.config import TFWENV
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
 
 
 class ServerDownlinkConnector(ZMQConnectorBase):
@@ -20,7 +23,10 @@ class ServerDownlinkConnector(ZMQConnectorBase):
 
         self.subscribe = partial(self._zmq_sub_socket.setsockopt_string, zmq.SUBSCRIBE)
         self.unsubscribe = partial(self._zmq_sub_socket.setsockopt_string, zmq.UNSUBSCRIBE)
-        self.register_callback = self._zmq_sub_stream.on_recv
+
+    def register_callback(self, callback):
+        callback = with_deserialize_tfw_msg(callback)
+        self._zmq_sub_stream.on_recv(callback)
 
 
 class ServerUplinkConnector(ZMQConnectorBase):
@@ -34,22 +40,40 @@ class ServerUplinkConnector(ZMQConnectorBase):
 
     def send_to_eventhandler(self, message):
         """
-        Send a message to an event handler.
+        Send a message to an event handler through the TFW server.
+
+        This envelopes the desired message in the 'data' field of the message to
+        TFWServer, which will mirror it to event handlers.
+
         :param message: JSON message you want to send
-        :param message['key']: key of event handler you want to address
         """
-        nested_message = {'key': message['key'], 'data': message.pop('data')}
-        message['key'] = 'mirror'
-        message['data'] = nested_message
-        self.send(message)
+        self.send({
+            'key': 'mirror',
+            'data': message
+        })
 
     def send(self, message):
         """
-        Send a message to the TFW server
+        Send a message to the frontend through the TFW server.
+
         :param message: JSON message you want to send
         """
         self._zmq_push_socket.send_multipart(serialize_tfw_msg(message))
 
+    def broadcast(self, message):
+        """
+        Broadast a message through the TFW server.
+
+        This envelopes the desired message in the 'data' field of the message to
+        TFWServer, which will broadast it.
+
+        :param message: JSON message you want to send
+        """
+        self.send({
+            'key': 'broadcast',
+            'data': message
+        })
+
 
 class ServerConnector(ServerUplinkConnector, ServerDownlinkConnector):
     pass

lib/tfw/networking/message_sender.py

@@ -1,30 +1,54 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from datetime import datetime
-
 from tfw.networking.event_handlers import ServerUplinkConnector
 
 
 class MessageSender:
     """
-    Provides a mechanism to send messages to our frontend messaging component which
-    displays messages with the key "message".
+    Provides mechanisms to send messages to our frontend messaging component.
     """
-    def __init__(self, custom_key: str = None):
+    def __init__(self):
         self.server_connector = ServerUplinkConnector()
-        self.key = custom_key or 'message'
+        self.key = 'message'
+        self.queue_key = 'queueMessages'
 
     def send(self, originator, message):
         """
-        Sends a message to the key specified in __init__.
+        Sends a message.
         :param originator: name of sender to be displayed on the frontend
         :param message: message to send
         """
         data = {
             'originator': originator,
-            'timestamp': datetime.now().isoformat(),
             'message': message
         }
-        self.server_connector.send({'key': self.key,
-                                    'data': data})
+        self.server_connector.send({
+            'key': self.key,
+            'data': data
+        })
+
+    def queue_messages(self, originator, messages):
+        """
+        Queues a list of messages to be displayed in a chatbot-like manner.
+        :param originator: name of sender to be displayed on the frontend
+        :param messages: list of messages to queue
+        """
+        data = {
+            'messages': [
+                {'message': message, 'originator': originator}
+                for message in messages
+            ]
+        }
+        self.server_connector.send({
+            'key': self.queue_key,
+            'data': data
+        })
+
+    @staticmethod
+    def generate_messages_from_queue(queue_message):
+        for message in queue_message['data']['messages']:
+            yield {
+                'key': 'message',
+                'data': message
+            }

lib/tfw/networking/serialization.py

@@ -22,35 +22,64 @@ The purpose of this module is abstracting away this low level behaviour.
 """
 
 import json
-
-
-def validate_message(message):
-    return 'key' in message
+from functools import wraps
 
 
 def serialize_tfw_msg(message):
+    """
+    Create TFW multipart data from message dict
+    """
     return _serialize_all(message['key'], message)
 
 
+def with_deserialize_tfw_msg(fun):
+    @wraps(fun)
+    def wrapper(message_parts):
+        message = deserialize_tfw_msg(*message_parts)
+        return fun(message)
+    return wrapper
+
+
 def deserialize_tfw_msg(*args):
+    """
+    Return message from TFW multipart data
+    """
     return _deserialize_all(*args)[1]
 
 
 def _serialize_all(*args):
-    return tuple(_serialize_single(arg) for arg in args)
+    return tuple(
+        _serialize_single(arg)
+        for arg in args
+    )
 
 
 def _deserialize_all(*args):
-    return tuple(_deserialize_single(arg) for arg in args)
+    return tuple(
+        _deserialize_single(arg)
+        for arg in args
+    )
 
 
 def _serialize_single(data):
+    """
+    Return input as bytes
+    (serialize input if it is JSON)
+    """
     if not isinstance(data, str):
-        data = json.dumps(data)
+        data = message_bytes(data)
     return _encode_if_needed(data)
 
 
+def message_bytes(message):
+    return json.dumps(message, sort_keys=True).encode()
+
+
 def _deserialize_single(data):
+    """
+    Try parsing input as JSON, return it as
+    string if parsing fails.
+    """
     try:
         return json.loads(data)
     except ValueError:
@@ -58,12 +87,20 @@ def _deserialize_single(data):
 
 
 def _encode_if_needed(value):
+    """
+    Return input as bytes
+    (encode if input is string)
+    """
     if isinstance(value, str):
         value = value.encode('utf-8')
     return value
 
 
 def _decode_if_needed(value):
+    """
+    Return input as string
+    (decode if input is bytes)
+    """
     if isinstance(value, (bytes, bytearray)):
         value = value.decode('utf-8')
     return value

lib/tfw/networking/server/__init__.py

@@ -3,5 +3,3 @@
 
 from .event_handler_connector import EventHandlerConnector, EventHandlerUplinkConnector, EventHandlerDownlinkConnector
 from .tfw_server import TFWServer
-from .zmq_websocket_handler import ZMQWebSocketProxy
-# from .controller_responder import ControllerResponder  # TODO: readd once controller stuff is resolved

lib/tfw/networking/server/controller_responder.py

@@ -1,38 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-from tfw.networking import deserialize_all, serialize_all, ControllerConnector
-
-
-class ControllerResponder:
-    def __init__(self, fsm):
-        self.fsm = fsm
-        self.token = None
-        self.controller_connector = ControllerConnector()
-        self.controller_connector.register_callback(self.handle_controller_request)
-        self.controller_request_handlers = {
-            'solution_check': self.handle_solution_check_request,
-            'test': self.handle_test_request,
-            'token': self.handle_token_request
-        }
-
-    def handle_controller_request(self, stream, msg_parts):
-        key, data = deserialize_all(*msg_parts)
-        response = self.controller_request_handlers[key](data)
-        stream.send_multipart(serialize_all(self.token, response))
-
-    def handle_test_request(self, data):
-        # pylint: disable=unused-argument,no-self-use
-        return 'OK'
-
-    def handle_token_request(self, data):
-        if self.token is None:
-            self.token = data
-        return {'token': self.token}
-
-    def handle_solution_check_request(self, data):
-        # pylint: disable=unused-argument
-        return {
-            'solved': self.fsm.is_solved(),
-            'message': 'solved' if self.fsm.is_solved() else 'not solved'
-        }

lib/tfw/networking/server/event_handler_connector.py

@@ -4,7 +4,7 @@
 import zmq
 from zmq.eventloop.zmqstream import ZMQStream
 
-from tfw.networking import ZMQConnectorBase, serialize_tfw_msg
+from tfw.networking import ZMQConnectorBase, serialize_tfw_msg, with_deserialize_tfw_msg
 from tfw.config import TFWENV
 from tfw.config.logs import logging
 
@@ -32,6 +32,7 @@ class EventHandlerUplinkConnector(ZMQConnectorBase):
 
 class EventHandlerConnector(EventHandlerDownlinkConnector, EventHandlerUplinkConnector):
     def register_callback(self, callback):
+        callback = with_deserialize_tfw_msg(callback)
         self._zmq_pull_stream.on_recv(callback)
 
     def send_message(self, message: dict):

lib/tfw/networking/server/tfw_server.py

@@ -1,15 +1,17 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from collections import defaultdict
+from abc import ABC, abstractmethod
+from contextlib import suppress
 
 from tornado.web import Application
 
-from tfw.networking import MessageSender
 from tfw.networking.event_handlers import ServerUplinkConnector
 from tfw.networking.server import EventHandlerConnector
+from tfw.networking import MessageSender
+from tfw.crypto import KeyManager, verify_message, sign_message
 from tfw.config.logs import logging
-from .zmq_websocket_handler import ZMQWebSocketProxy
+from .zmq_websocket_proxy import ZMQWebSocketProxy
 
 LOG = logging.getLogger(__name__)
 
@@ -18,103 +20,83 @@ class TFWServer:
     """
     This class handles the proxying of messages between the frontend and event handers.
     It proxies messages from the "/ws" route to all event handlers subscribed to a ZMQ
-    SUB socket. It also manages an FSM you can define as a constructor argument.
+    SUB socket.
     """
-    def __init__(self, fsm_type):
-        """
-        :param fsm_type: the type of FSM you want TFW to use
-        """
-        self._fsm = fsm_type()
-        self._fsm_updater = FSMUpdater(self._fsm)
-        self._fsm_manager = FSMManager(self._fsm)
-        self._fsm.subscribe_callback(self._fsm_updater.update)
+    def __init__(self):
         self._event_handler_connector = EventHandlerConnector()
+        self._uplink_connector = ServerUplinkConnector()
+        self._auth_key = KeyManager().auth_key
 
-        self.application = Application(
-            [(r'/ws', ZMQWebSocketProxy, {'make_eventhandler_message': self.make_eventhandler_message,
-                                          'proxy_filter': self.proxy_filter,
-                                          'handle_trigger': self.handle_trigger,
-                                          'event_handler_connector': self._event_handler_connector})]
-        )
-        #self.controller_responder = ControllerResponder(self.fsm) TODO: add this once controller stuff is resolved
+        self.application = Application([(
+            r'/ws', ZMQWebSocketProxy, {
+                'event_handler_connector': self._event_handler_connector,
+                'message_handlers': [self.handle_trigger, self.handle_recover],
+                'frontend_message_handlers': [self.save_frontend_messages]
+            })])
 
-    @property
-    def fsm(self):
-        return self._fsm
-
-    @property
-    def fsm_manager(self):
-        return self._fsm_manager
-
-    def make_eventhandler_message(self, message):
-        self.trigger_fsm(message)
-        message['FSMUpdate'] = self._fsm_updater.get_fsm_state_and_transitions()
-        return message
+        self._frontend_messages = FrontendMessageStorage()
 
     def handle_trigger(self, message):
-        LOG.debug('Executing handler for trigger "%s"', message.get('trigger', ''))
-        self.trigger_fsm(message)
+        if 'trigger' in message:
+            LOG.debug('Executing handler for trigger "%s"', message.get('trigger', ''))
+            fsm_eh_command = {
+                'key': 'fsm',
+                'data': {
+                    'command': 'trigger',
+                    'value': message['trigger']
+                }
+            }
+            if verify_message(self._auth_key, message):
+                sign_message(self._auth_key, fsm_eh_command)
+            self._uplink_connector.send_to_eventhandler(fsm_eh_command)
 
-    def trigger_fsm(self, message):
-        trigger = message.get('trigger', '')
-        try:
-            self._fsm_manager.trigger(trigger, message)
-        except AttributeError:
-            LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
+    def handle_recover(self, message):
+        if message['key'] == 'recover':
+            self._frontend_messages.replay_messages(self._uplink_connector)
+            self._frontend_messages.clear()
 
-    def proxy_filter(self, message):
-        # pylint: disable=unused-argument,no-self-use
-        return True
+    def save_frontend_messages(self, message):
+        self._frontend_messages.save_message(message)
 
     def listen(self, port):
         self.application.listen(port)
 
 
-class FSMManager:
-    def __init__(self, fsm):
-        self._fsm = fsm
-        self.trigger_predicates = defaultdict(list)
-        self.messenge_sender = MessageSender()
+class MessageStorage(ABC):
+    def __init__(self):
+        self.saved_messages = []
 
-    @property
-    def fsm(self):
-        return self._fsm
+    def save_message(self, message):
+        with suppress(KeyError, AttributeError):
+            if self.filter_message(message):
+                self.saved_messages.extend(self.transform_message(message))
 
-    def trigger(self, trigger, message):
-        predicate_results = []
-        for predicate in self.trigger_predicates[trigger]:
-            success, message = predicate(message)
-            predicate_results.append(success)
-            self.messenge_sender.send('FSM', message)
+    @abstractmethod
+    def filter_message(self, message):
+        raise NotImplementedError
 
-        if all(predicate_results):
-            try:
-                self.fsm.trigger(trigger, message=message)
-            except AttributeError:
-                LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
+    def transform_message(self, message):   # pylint: disable=no-self-use
+        yield message
 
-    def subscribe_predicate(self, trigger, *predicates):
-        self.trigger_predicates[trigger].extend(predicates)
-
-    def unsubscribe_predicate(self, trigger, *predicates):
-        self.trigger_predicates[trigger] = [predicate for predicate in self.trigger_predicates[trigger]
-                                            not in predicates]
+    def clear(self):
+        self.saved_messages.clear()
 
 
-class FSMUpdater:
-    def __init__(self, fsm):
-        self.fsm = fsm
-        self.uplink = ServerUplinkConnector()
+class FrontendMessageStorage(MessageStorage):
+    def filter_message(self, message):
+        key = message['key']
+        command = message.get('data', {}).get('command')
+        return (
+            key in ('message', 'dashboard', 'queueMessages')
+            or key == 'ide' and command in ('select', 'read')
+        )
 
-    def update(self, kwargs_dict):
-        # pylint: disable=unused-argument
-        self.uplink.send(self.generate_fsm_update())
+    def transform_message(self, message):
+        if message['key'] == 'queueMessages':
+            yield from MessageSender.generate_messages_from_queue(message)
+        else:
+            yield message
 
-    def generate_fsm_update(self):
-        return {'key': 'FSMUpdate',
-                'data': self.get_fsm_state_and_transitions()}
-
-    def get_fsm_state_and_transitions(self):
-        state = self.fsm.state
-        valid_transitions = [{'trigger': trigger} for trigger in self.fsm.machine.get_triggers(self.fsm.state)]
-        return {'current_state': state, 'valid_transitions': valid_transitions}
+    def replay_messages(self, connector):
+        for message in self.saved_messages:
+            connector.send(message)

lib/tfw/networking/server/zmq_websocket_handler.py

@@ -1,91 +0,0 @@
-# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-# All Rights Reserved. See LICENSE file for details.
-
-import json
-from abc import ABC, abstractmethod
-
-from tornado.websocket import WebSocketHandler
-
-from tfw.networking import deserialize_tfw_msg, validate_message
-from tfw.config.logs import logging
-
-LOG = logging.getLogger(__name__)
-
-
-class ZMQWebSocketHandler(WebSocketHandler, ABC):
-    instances = set()
-
-    def initialize(self, **kwargs): # pylint: disable=arguments-differ
-        self._event_handler_connector = kwargs['event_handler_connector']
-
-    def prepare(self):
-        ZMQWebSocketHandler.instances.add(self)
-
-    def on_close(self):
-        ZMQWebSocketHandler.instances.remove(self)
-
-    def open(self, *args, **kwargs):
-        LOG.debug('WebSocket connection initiated')
-        self._event_handler_connector.register_callback(self.zmq_callback)
-
-    def zmq_callback(self, msg_parts):
-        keyhandlers = {'mirror': self.mirror}
-
-        message = deserialize_tfw_msg(*msg_parts)
-        LOG.debug('Received on pull socket: %s', message)
-        if not validate_message(message):
-            return
-
-        self.handle_trigger(message)
-        if message['key'] not in keyhandlers:
-            for instance in ZMQWebSocketHandler.instances:
-                instance.write_message(message)
-        else:
-            try:
-                keyhandlers[message['key']](message)
-            except KeyError:
-                LOG.error('Invalid mirror message format! Ignoring.')
-
-    def mirror(self, message):
-        message = message['data']
-        self._event_handler_connector.send_message(message)
-
-    def on_message(self, message):
-        LOG.debug('Received on WebSocket: %s', message)
-        if validate_message(message):
-            self.send_message(self.make_eventhandler_message(message))
-
-    @abstractmethod
-    def make_eventhandler_message(self, message):
-        raise NotImplementedError
-
-    def send_message(self, message: dict):
-        self._event_handler_connector.send_message(message)
-
-    @abstractmethod
-    def handle_trigger(self, message):
-        raise NotImplementedError
-
-    # much secure, very cors, wow
-    def check_origin(self, origin):
-        return True
-
-
-class ZMQWebSocketProxy(ZMQWebSocketHandler):
-    # pylint: disable=abstract-method
-    def initialize(self, **kwargs): # pylint: disable=arguments-differ
-        super(ZMQWebSocketProxy, self).initialize(**kwargs)
-        self._make_eventhandler_message = kwargs['make_eventhandler_message']
-        self._proxy_filter = kwargs['proxy_filter']
-        self._handle_trigger = kwargs['handle_trigger']
-
-    def on_message(self, message):
-        message = json.loads(message)
-        if self._proxy_filter(message):
-            super().on_message(message)
-
-    def make_eventhandler_message(self, message):
-        return self._make_eventhandler_message(message)
-
-    def handle_trigger(self, message):
-        self._handle_trigger(message)

lib/tfw/networking/server/zmq_websocket_proxy.py

@@ -0,0 +1,138 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+import json
+
+from tornado.websocket import WebSocketHandler
+
+from tfw.mixins import CallbackMixin
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class ZMQWebSocketProxy(WebSocketHandler):
+    # pylint: disable=abstract-method
+    instances = set()
+
+    def initialize(self, **kwargs): # pylint: disable=arguments-differ
+        self._event_handler_connector = kwargs['event_handler_connector']
+
+        self._message_handlers = kwargs.get('message_handlers', [])
+        self._frontend_message_handlers = kwargs.get('frontend_message_handlers', [])
+        self._eventhandler_message_handlers = kwargs.get('eventhandler_message_handlers', [])
+        self._proxy_filters = kwargs.get('proxy_filters', [])
+
+        self.proxy_eventhandler_to_websocket = TFWProxy(
+            self.send_eventhandler_message,
+            self.send_websocket_message
+        )
+        self.proxy_websocket_to_eventhandler = TFWProxy(
+            self.send_websocket_message,
+            self.send_eventhandler_message
+        )
+
+        self.subscribe_proxy_callbacks()
+
+    def subscribe_proxy_callbacks(self):
+        self.proxy_websocket_to_eventhandler.subscribe_proxy_callbacks_and_filters(
+            self._eventhandler_message_handlers + self._message_handlers,
+            self._proxy_filters
+        )
+
+        self.proxy_eventhandler_to_websocket.subscribe_proxy_callbacks_and_filters(
+            self._frontend_message_handlers + self._message_handlers,
+            self._proxy_filters
+        )
+
+    def prepare(self):
+        ZMQWebSocketProxy.instances.add(self)
+
+    def on_close(self):
+        ZMQWebSocketProxy.instances.remove(self)
+
+    def open(self, *args, **kwargs):
+        LOG.debug('WebSocket connection initiated')
+        self._event_handler_connector.register_callback(self.eventhander_callback)
+
+    def eventhander_callback(self, message):
+        """
+        Invoked on ZMQ messages from event handlers.
+        """
+        LOG.debug('Received on pull socket: %s', message)
+        self.proxy_eventhandler_to_websocket(message)
+
+    def on_message(self, message):
+        """
+        Invoked on WS messages from frontend.
+        """
+        message = json.loads(message)
+        LOG.debug('Received on WebSocket: %s', message)
+        self.proxy_websocket_to_eventhandler(message)
+
+    def send_eventhandler_message(self, message):
+        self._event_handler_connector.send_message(message)
+
+    @staticmethod
+    def send_websocket_message(message):
+        for instance in ZMQWebSocketProxy.instances:
+            instance.write_message(message)
+
+    # much secure, very cors, wow
+    def check_origin(self, origin):
+        return True
+
+
+class TFWProxy:
+    # pylint: disable=protected-access
+    def __init__(self, to_source, to_destination):
+        self.to_source = to_source
+        self.to_destination = to_destination
+
+        self.proxy_filters = CallbackMixin()
+        self.proxy_callbacks = CallbackMixin()
+
+        self.proxy_filters.subscribe_callback(self.validate_message)
+
+        self.keyhandlers = {
+            'mirror':    self.mirror,
+            'broadcast': self.broadcast
+        }
+
+    @staticmethod
+    def validate_message(message):
+        if 'key' not in message:
+            raise ValueError('Invalid TFW message format!')
+
+    def __call__(self, message):
+        try:
+            self.proxy_filters._execute_callbacks(message)
+        except ValueError:
+            LOG.exception('Invalid TFW message received!')
+            return
+
+        self.proxy_callbacks._execute_callbacks(message)
+
+        if message['key'] not in self.keyhandlers:
+            self.to_destination(message)
+        else:
+            handler = self.keyhandlers[message['key']]
+            try:
+                handler(message)
+            except KeyError:
+                LOG.error('Invalid "%s" message format! Ignoring.', handler.__name__)
+
+    def mirror(self, message):
+        message = message['data']
+        LOG.debug('Mirroring message: %s', message)
+        self.to_source(message)
+
+    def broadcast(self, message):
+        message = message['data']
+        LOG.debug('Broadcasting message: %s', message)
+        self.to_source(message)
+        self.to_destination(message)
+
+    def subscribe_proxy_callbacks_and_filters(self, proxy_callbacks, proxy_filters):
+        self.proxy_callbacks.subscribe_callbacks(*proxy_callbacks)
+        self.proxy_filters.subscribe_callbacks(*proxy_filters)

lib/tfw/yaml_fsm.py

@@ -0,0 +1,95 @@
+from subprocess import Popen, run
+from functools import partial, singledispatch
+from contextlib import suppress
+
+import yaml
+import jinja2
+from transitions import State
+
+from tfw import FSMBase
+
+
+class YamlFSM(FSMBase):
+    def __init__(self, config_file, jinja2_variables=None):
+        self.config = ConfigParser(config_file, jinja2_variables).config
+        self.setup_states()
+        super().__init__()  # FSMBase.__init__() requires states
+        self.setup_transitions()
+
+    def setup_states(self):
+        self.for_config_states_and_transitions_do(self.wrap_callbacks_with_subprocess_call)
+        self.states = [State(**state) for state in self.config['states']]
+
+    def setup_transitions(self):
+        self.for_config_states_and_transitions_do(self.subscribe_and_remove_predicates)
+        for transition in self.config['transitions']:
+            self.add_transition(**transition)
+
+    def for_config_states_and_transitions_do(self, what):
+        for array in ('states', 'transitions'):
+            for json_obj in self.config[array]:
+                what(json_obj)
+
+    @staticmethod
+    def wrap_callbacks_with_subprocess_call(json_obj):
+        topatch = ('on_enter', 'on_exit', 'prepare', 'before', 'after')
+        for key in json_obj:
+            if key in topatch:
+                json_obj[key] = partial(run_command_async, json_obj[key])
+
+    def subscribe_and_remove_predicates(self, json_obj):
+        if 'predicates' in json_obj:
+            for predicate in json_obj['predicates']:
+                self.subscribe_predicate(
+                    json_obj['trigger'],
+                    partial(
+                        command_statuscode_is_zero,
+                        predicate
+                        )
+                )
+
+        with suppress(KeyError):
+            json_obj.pop('predicates')
+
+
+def run_command_async(command, _):
+    Popen(command, shell=True)
+
+
+def command_statuscode_is_zero(command):
+    return run(command, shell=True).returncode == 0
+
+
+class ConfigParser:
+    def __init__(self, config_file, jinja2_variables):
+        self.read_variables = singledispatch(self._read_variables)
+        self.read_variables.register(dict, self._read_variables_dict)
+        self.read_variables.register(str, self._read_variables_str)
+
+        self.config = self.parse_config(config_file, jinja2_variables)
+
+    def parse_config(self, config_file, jinja2_variables):
+        config_string = self.read_file(config_file)
+        if jinja2_variables is not None:
+            variables = self.read_variables(jinja2_variables)
+            template = jinja2.Environment(loader=jinja2.BaseLoader).from_string(config_string)
+            config_string = template.render(**variables)
+        return yaml.safe_load(config_string)
+
+    @staticmethod
+    def read_file(filename):
+        with open(filename, 'r') as ifile:
+            return ifile.read()
+
+    @staticmethod
+    def _read_variables(variables):
+        raise TypeError(f'Invalid variables type {type(variables)}')
+
+    @staticmethod
+    def _read_variables_str(variables):
+        with open(variables, 'r') as ifile:
+            return yaml.safe_load(ifile)
+
+    @staticmethod
+    def _read_variables_dict(variables):
+        return variables

nginx/default.conf

@@ -4,6 +4,7 @@ server {
     proxy_connect_timeout 7d;
     proxy_send_timeout 7d;
     proxy_read_timeout 7d;
+    absolute_redirect off;
 
     location = /ws {
         proxy_pass http://127.0.0.1:${TFW_WEB_PORT};

requirements.txt

@@ -3,3 +3,6 @@ pyzmq==17.0.0
 transitions==0.6.4
 terminado==0.8.1
 watchdog==0.8.3
+PyYAML==3.12
+Jinja2==2.10
+cryptography==2.2.2

setup.py

@@ -0,0 +1,29 @@
+from os.path import dirname, realpath, join
+
+from setuptools import setup, find_packages
+
+here = dirname(realpath(__file__))
+
+with open(join(here, 'VERSION'), 'r') as ifile:
+    version = ifile.read().strip('\n')
+with open(join(here, 'requirements.txt'), 'r') as ifile:
+    requirements = ifile.read().splitlines()
+
+setup(
+    name = 'tfw',
+    version = version,
+    description = 'Avatao tutorial-framework',
+    url = 'https://github.com/avatao-content/baseimage-tutorial-framework',
+    author = 'Avatao.com Innovative Learning Kft.',
+    author_email = 'support@avatao.com',
+    license = 'custom',
+    packages = find_packages('lib'),
+    package_dir = {'': 'lib'},
+    install_requires = requirements,
+    extras_require = {
+        'docs': [
+            'sphinx >= 1.7.0',
+        ],
+    },
+    zip_safe = False,
+)

supervisor/components/tfw_server.conf

@@ -0,0 +1,4 @@
+[program:tfwserver]
+user=root
+directory=%(ENV_TFW_SERVER_DIR)s
+command=python3 tfw_server.py

supervisor/supervisord.conf

@@ -1,8 +1,10 @@
 [supervisord]
 user=root
-logfile = /tmp/supervisord.log
+logfile=/dev/null
+logfile_maxbytes=0
 loglevel = debug
 pidfile = /tmp/supervisord.pid
+strip_ansi=true
 
 [inet_http_server]
 port = 127.0.0.1:%(ENV_TFW_SUPERVISOR_HTTP_PORT)s

supervisor/tfw_server.py

@@ -0,0 +1,9 @@
+from tornado.ioloop import IOLoop
+
+from tfw.networking import TFWServer
+from tfw.config import TFWENV
+
+
+if __name__ == '__main__':
+    TFWServer().listen(TFWENV.WEB_PORT)
+    IOLoop.instance().start()