Add .drone.yml

Networking refactor

.drone.yml

@@ -0,0 +1,13 @@
+pipeline:
+  build:
+    image: eu.gcr.io/avatao-public/docker:el7
+    volumes:
+      - /etc/docker:/etc/docker:ro
+      - /root/.docker:/root/.docker:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    commands:
+      - docker build --pull -t eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG} .
+      - docker push eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG}
+    when:
+      event: 'tag'
+      branch: refs/tags/egyptianmau-20*

.pyenvrc

@@ -1,4 +0,0 @@
-export PYENV_ROOT="$HOME/.pyenv"
-export PATH="$PYENV_ROOT/bin:$PATH"
-eval "$(pyenv init -)"
-eval "$(pyenv virtualenv-init -)"

.pylintrc

@@ -0,0 +1,5 @@
+[TYPECHECK]
+
+ignored-modules = zmq
+max-line-length = 150
+disable = missing-docstring, too-few-public-methods

Dockerfile

@@ -53,16 +53,20 @@ COPY nginx/nginx.conf ${TFW_NGINX_CONF}
 COPY nginx/default.conf ${TFW_NGINX_DEFAULT}
 COPY lib ${TFW_LIB_DIR}
 
-ONBUILD ARG BUILD_CONTEXT="."
+RUN for dir in "${TFW_LIB_DIR}" "/etc/nginx" "/etc/supervisor"; do \
+        chown -R root:root "$dir" && chmod -R 700 "$dir";          \
+    done
+
+ONBUILD ARG BUILD_CONTEXT="solvable"
 ONBUILD ARG NOFRONTEND=""
 
 ONBUILD COPY ${BUILD_CONTEXT}/nginx/components/ ${TFW_NGINX_COMPONENTS}
 ONBUILD COPY ${BUILD_CONTEXT}/supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS}
 
-ONBUILD RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx                                          &&\
+ONBUILD RUN for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do                               \
-            for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do                               \
                 envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f~ && mv $f~ $f ;\
             done
+ONBUILD VOLUME ["/etc/nginx", "/var/lib/nginx", "/var/log/nginx"]
 
 ONBUILD COPY ${BUILD_CONTEXT}/frontend /data/
 ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn install --frozen-lockfile || :

LICENSE

@@ -0,0 +1,12 @@
+AVATAO CONFIDENTIAL
+
+Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+All Rights Reserved.
+
+All source code, configuration files and documentation contained herein
+is, and remains the exclusive property of Avatao.com Innovative Learning Kft.
+The intellectual and technical concepts contained herein are proprietary 
+to Avatao.com Innovative Learning Kft. and are protected by trade secret
+or copyright law. Dissemination of this information or reproduction of
+this material is strictly forbidden unless prior written permission is
+obtained from Avatao.com Innovative Learning Kft.

README.md

@@ -1,16 +1,49 @@
-Tutorial Framework
+# baseimage-tutorial-framework
-==================
 
-## Building and running with magic starter
+This is the beating heart of TFW – the Docker baseimage containing the internals of the framework.
 
-Execute the `tfw_magic_start.sh` script and it will automagically run a backend instance in Docker and serve the Angular frontend locally.
+All tutorial-framework challenges are child images of this one: their `Dockerfile`s all begin with `FROM avatao/tutorial-framework`.
 
-This requires the `test-tutorial-framework` (Docker image `FROM baseimage-tutorial-framework`) and `frontend-tutorial-framework` repositories to have a common parent folder.
+This document explains the general concepts of TFW and should be the first thing you read before getting started with development.
 
-## Building with Docker
+For more on building and running you should consult the `test-tutorial-framework` repo.
 
-Simply issue `docker build -t baseimage-tutorial-framework .` in the project root.
+## The framework
 
-## Creating child images
+The goal of the tutorial-framework is to help content developers in creating interactive tutorials for the Avatao platform.
 
-Documentation in progress...
+To make this possible TFW implements a robust messaging system and provides several pre-written components built upon it, such as a file editor or a terminal (running in your browser).
+
+The foundation of the whole framework is the messaging system connecting the frontend with the backend.
+Frontend components use websockets to connect to TFW, to which you can hook several *event handlers* defining how to handle specific messages.
+
+![TFW architecture](docs/tfw_architecture.png)
+
+### Event handlers
+
+Imagine event handlers as callbacks that are executed when TFW receives a specific type of message. For example you could send a message to the framework when the user does something of note.
+
+This allows you to define actions triggered on the backend when the user presses a button on the frontend, moves the cursor to a specific area or anything like that.
+
+Event handlers use ZeroMQ to connect to the framework. They are as loosely-coupled as possible: usually they are running in separate processes and only communicate with TFW through ZMQ.
+
+Most of pre-made event handlers are writen in Python3, but you can write event handlers in any language that has ZeroMQ bindings (this means virtually any language).
+
+This makes the framework really flexible: you can demonstrate the concepts you want to in any language while using the same set of tools provided by TFW.
+
+### FSM
+
+Another unique feature of the framework is the FSM – finite state machine – representing the state of your challenge.
+This allows you to track users progressing with the tasks you've defined for them to complete.
+
+For instance you could represent whether the user managed to create a malicious user with a state called `user_registered` and subscribe callbacks to events regarding that state (like entering or leaving).
+
+You could create challenges that can be completed in several different ways: imagine a state called `challenge_complete`, which represents when the challenge is completed. Several series of actions could lead to this state.
+
+This enables you to guide your users through the experience you've envisioned with your tutorial.
+We can provide a whole new level of interactivity in our challenges because we know what the user is doing.
+This includes context-dependent hints and the automatic typing of commands to a terminal.
+
+### Frontend
+
+Note that our frontend implementation is written in Angular. It is maintained and documented in the `frontend-tutorial-framework` repository.

lib/envvars.py

@@ -1,3 +1,6 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from collections import namedtuple
 from os import environ
 

lib/tao/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.

lib/tao/config/__init__.py

@@ -1 +1,4 @@
-from .envvars import * # pylint: disable=wildcard-import
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from .envvars import TAOENV

lib/tao/config/envvars.py

@@ -1,3 +1,6 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from envvars import prefixed_envvars_to_namedtuple
 
 TAOENV = prefixed_envvars_to_namedtuple('AVATAO_', 'taoenvtuple')

lib/tfw/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from .event_handler_base import TriggerlessEventHandler, TriggeredEventHandler
+from .fsm_base import FSMBase
+from .linear_fsm import LinearFSM

lib/tfw/components/__init__.py

@@ -0,0 +1,9 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from .directory_monitoring_event_handler import DirectoryMonitoringEventHandler
+from .process_managing_event_handler import ProcessManagingEventHandler
+from .terminado_event_handler import TerminadoEventHandler
+from .webide_event_handler import WebideEventHandler
+from .history_monitor import HistoryMonitor, BashMonitor, GDBMonitor
+from .terminal_commands import TerminalCommands

lib/tfw/components/decorators/__init__.py

@@ -1 +0,0 @@
-from .rate_limiter import RateLimiter

lib/tfw/components/directory_monitor.py

@@ -1,30 +1,26 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from functools import wraps
 
-from watchdog.observers import Observer
 from watchdog.events import FileSystemEventHandler as FileSystemWatchdogEventHandler
 
-from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
+from tfw.networking.event_handlers import ServerUplinkConnector
-from tfw.components.decorators import RateLimiter
+from tfw.decorators import RateLimiter
+from tfw.mixins import ObserverMixin
 
 from tfw.config.logs import logging
 
 LOG = logging.getLogger(__name__)
 
 
-class DirectoryMonitor:
+class DirectoryMonitor(ObserverMixin):
     def __init__(self, directory):
-        self.observer = Observer()
+        ObserverMixin.__init__(self)
         self.eventhandler = WebideReloadWatchdogEventHandler()
         self.observer.schedule(self.eventhandler, directory, recursive=True)
         self.pause, self.resume = self.eventhandler.pause, self.eventhandler.resume
 
-    def watch(self):
-        self.observer.start()
-
-    def stop(self):
-        self.observer.stop()
-        self.observer.join()
-
     @property
     def ignore(self):
         return self.eventhandler.ignore
@@ -67,8 +63,8 @@ class WebideReloadWatchdogEventHandler(FileSystemWatchdogEventHandler):
             self.ignore = self.ignore - 1
             return
         LOG.debug(event)
-        key = 'webide'
+        self.uplink.send({'key': 'webide',
-        self.uplink.send(key, {'data': {'command': 'reload'}})
+                          'data': {'command': 'reload'}})
 
 
 def with_monitor_paused(fun):

lib/tfw/components/directory_monitoring_event_handler.py

@@ -1,18 +1,22 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from os.path import isdir, exists
 
-from tfw.components.directory_monitor import DirectoryMonitor
+from tfw import TriggerlessEventHandler
-from tfw.event_handler_base import TriggerlessEventHandler
 from tfw.config.logs import logging
+from tfw.mixins import MonitorManagerMixin
+from .directory_monitor import DirectoryMonitor
 
 LOG = logging.getLogger(__name__)
 
 
-class DirectoryMonitoringEventHandler(TriggerlessEventHandler):
+class DirectoryMonitoringEventHandler(TriggerlessEventHandler, MonitorManagerMixin):
     def __init__(self, key, directory):
         super().__init__(key)
         self._directory = directory
-        self._monitor = None
+        MonitorManagerMixin.__init__(self, DirectoryMonitor, self._directory)
-        self.reload_monitor()
+
         self.commands = {'pause':     self.pause,
                          'resume':    self.resume,
                          'ignore':    self.ignore,
@@ -28,20 +32,7 @@ class DirectoryMonitoringEventHandler(TriggerlessEventHandler):
             raise EnvironmentError('No such directory!')
         self._directory = directory
 
-    @property
+    def handle_event(self, message):
-    def monitor(self):
-        return self._monitor
-
-    def reload_monitor(self):
-        if self._monitor:
-            try:
-                self._monitor.stop()
-            except KeyError:
-                logging.debug('Working directory was removed – ignoring...')
-        self._monitor = DirectoryMonitor(self._directory)
-        self._monitor.watch()   # This runs on a separate thread
-
-    def handle_event(self, key, message):
         try:
             message['data'] = self.commands[message['data']['command']](message['data'])
             return message

lib/tfw/components/history_monitor.py

@@ -1,13 +1,15 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from os.path import dirname
 from re import findall
 from re import compile as compileregex
 from abc import ABC, abstractmethod
 
-from watchdog.observers import Observer
 from watchdog.events import PatternMatchingEventHandler
 
-from tfw.components.mixins.callback_mixin import CallbackMixin
+from tfw.mixins import CallbackMixin, ObserverMixin
-from tfw.components.decorators.rate_limiter import RateLimiter
+from tfw.decorators import RateLimiter
 
 
 class CallbackEventHandler(PatternMatchingEventHandler, ABC):
@@ -21,13 +23,13 @@ class CallbackEventHandler(PatternMatchingEventHandler, ABC):
             callback()
 
 
-class HistoryMonitor(CallbackMixin, ABC):
+class HistoryMonitor(CallbackMixin, ObserverMixin, ABC):
     def __init__(self, histfile):
         CallbackMixin.__init__(self)
+        ObserverMixin.__init__(self)
         self.histfile = histfile
         self._history = []
         self._last_length = len(self._history)
-        self.observer = Observer()
         self.observer.schedule(CallbackEventHandler([self.histfile],
                                                     self._fetch_history,
                                                     self._invoke_callbacks),
@@ -57,13 +59,6 @@ class HistoryMonitor(CallbackMixin, ABC):
         if self._last_length < len(self._history):
             self._execute_callbacks(self.history)
 
-    def watch(self):
-        self.observer.start()
-
-    def stop(self):
-        self.observer.stop()
-        self.observer.join()
-
 
 class BashMonitor(HistoryMonitor):
     @property

lib/tfw/components/mixins/__init__.py

@@ -1 +0,0 @@
-from .supervisor_mixin import SupervisorMixin

lib/tfw/components/process_managing_event_handler.py

@@ -1,9 +1,12 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from xmlrpc.client import Fault as SupervisorFault
 
-from tfw.event_handler_base import TriggerlessEventHandler
+from tfw import TriggerlessEventHandler
-from tfw.components.mixins import SupervisorMixin
+from tfw.mixins import SupervisorMixin
-from tfw.components.directory_monitor import with_monitor_paused
 from tfw.config.logs import logging
+from .directory_monitor import with_monitor_paused
 
 LOG = logging.getLogger(__name__)
 
@@ -27,7 +30,7 @@ class ProcessManagingEventHandler(TriggerlessEventHandler):
         self.processmanager = ProcessManager()
 
     @with_monitor_paused
-    def handle_event(self, key, message):
+    def handle_event(self, message):
         try:
             data = message['data']
             self.processmanager(data['command'], data['process_name'])

lib/tfw/components/terminado_event_handler.py

@@ -1,7 +1,11 @@
-from tfw.components.terminado_mini_server import TerminadoMiniServer
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-from tfw.event_handler_base import TriggerlessEventHandler
+# All Rights Reserved. See LICENSE file for details.
+
+from tfw import TriggerlessEventHandler
 from tfw.config import TFWENV
 from tfw.config.logs import logging
+from tao.config import TAOENV
+from .terminado_mini_server import TerminadoMiniServer
 
 LOG = logging.getLogger(__name__)
 
@@ -11,7 +15,8 @@ class TerminadoEventHandler(TriggerlessEventHandler):
         super().__init__(key)
         self.working_directory = TFWENV.TERMINADO_DIR
         self._historymonitor = monitor
-        self.terminado_server = TerminadoMiniServer('/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, ['bash'])
+        bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash']
+        self.terminado_server = TerminadoMiniServer('/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, bash_as_user_cmd)
         self.commands = {'write': self.write,
                          'read': self.read}
         if self._historymonitor:
@@ -22,10 +27,11 @@ class TerminadoEventHandler(TriggerlessEventHandler):
     def historymonitor(self):
         return self._historymonitor
 
-    def handle_event(self, key, message):
+    def handle_event(self, message):
         LOG.debug('TerminadoEventHandler received event: %s', message)
         try:
-            message['data'] = self.commands[message['data']['command']](message['data'])
+            data = message['data']
+            message['data'] = self.commands[data['command']](data)
             return message
         except KeyError:
             LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)

lib/tfw/components/terminado_mini_server.py

@@ -1,3 +1,6 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from tornado.ioloop import IOLoop
 from tornado.web import Application
 from terminado import TermSocket, SingleTermManager

lib/tfw/components/terminal_commands.py

@@ -0,0 +1,48 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from abc import ABC
+from re import match
+
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class TerminalCommands(ABC):
+    def __init__(self, bashrc=None):
+        self._command_method_regex = r'^command_(.+)$'
+        self.command_implemetations = self._build_command_to_implementation_dict()
+        if bashrc is not None:
+            self._setup_bashrc_aliases(bashrc)
+
+    def _build_command_to_implementation_dict(self):
+        return {self._parse_command_name(fun): getattr(self, fun) for fun in dir(self)
+                if callable(getattr(self, fun)) and self._is_command_implementation(fun)}
+
+    def _setup_bashrc_aliases(self, bashrc):
+        with open(bashrc, 'a') as ofile:
+            alias_template = 'alias {0}="{0} > /dev/null 2>&1"\n'
+            for command in self.command_implemetations.keys():
+                ofile.write(alias_template.format(command))
+
+    def _is_command_implementation(self, method_name):
+        return bool(self._match_command_regex(method_name))
+
+    def _parse_command_name(self, method_name):
+        try:
+            return self._match_command_regex(method_name).groups()[0]
+        except AttributeError:
+            return ''
+
+    def _match_command_regex(self, string):
+        return match(self._command_method_regex, string)
+
+    def callback(self, history):
+        parts = history[-1].split()
+        command = parts[0]
+        if command in self.command_implemetations.keys():
+            try:
+                self.command_implemetations[command](*parts[1:])
+            except IndexError:
+                LOG.debug('Command "%s" failed!', command)

lib/tfw/components/webide_event_handler.py

@@ -1,18 +1,23 @@
-from os.path import isfile, join, relpath, exists, isdir
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from os.path import isfile, join, relpath, exists, isdir, realpath
 from glob import glob
 from fnmatch import fnmatchcase
 from collections import Iterable
 
-from tfw.event_handler_base import TriggerlessEventHandler
+from tfw import TriggerlessEventHandler
-from tfw.components.directory_monitor import DirectoryMonitor
+from tfw.mixins import MonitorManagerMixin
 from tfw.config.logs import logging
+from .directory_monitor import DirectoryMonitor
 
 LOG = logging.getLogger(__name__)
 
 
-class FileManager:
+class FileManager:  # pylint: disable=too-many-instance-attributes
-    def __init__(self, working_directory, selected_file=None, exclude=None):
+    def __init__(self, working_directory, allowed_directories, selected_file=None, exclude=None):
         self._exclude, self.exclude = None, exclude
+        self._allowed_directories, self.allowed_directories = None, allowed_directories
         self._workdir, self.workdir = None, working_directory
         self._filename, self.filename = None, selected_file or self.files[0]
 
@@ -36,23 +41,32 @@ class FileManager:
     def workdir(self, directory):
         if not exists(directory) or not isdir(directory):
             raise EnvironmentError('"{}" is not a directory!'.format(directory))
+        if not self._is_in_whitelisted_dir(directory):
+            raise EnvironmentError('Directory "{}" is not in whitelist!'.format(directory))
         self._workdir = directory
 
+    @property
+    def allowed_directories(self):
+        return self._allowed_directories
+
+    @allowed_directories.setter
+    def allowed_directories(self, directories):
+        self._allowed_directories = directories
+
     @property
     def filename(self):
         return self._filename
 
     @filename.setter
     def filename(self, filename):
-        if not filename in self.files:
+        if filename not in self.files:
             raise EnvironmentError('No such file in workdir!')
         self._filename = filename
 
     @property
     def files(self):
         return [self._relpath(file) for file in glob(join(self._workdir, '**/*'), recursive=True)
-                if isfile(file) and
+                if isfile(file) and self._is_in_whitelisted_dir(file) and not self._is_blacklisted(file)]
-                not any(fnmatchcase(file, blacklisted) for blacklisted in self.exclude)]
 
     @property
     def file_contents(self):
@@ -64,6 +78,12 @@ class FileManager:
         with open(self._filepath(self.filename), 'w', errors='surrogateescape') as ofile:
             ofile.write(value)
 
+    def _is_in_whitelisted_dir(self, path):
+        return any(realpath(path).startswith(allowed_dir) for allowed_dir in self.allowed_directories)
+
+    def _is_blacklisted(self, file):
+        return any(fnmatchcase(file, blacklisted) for blacklisted in self.exclude)
+
     def _filepath(self, filename):
         return join(self._workdir, filename)
 
@@ -71,10 +91,13 @@ class FileManager:
         return relpath(self._filepath(filename), start=self._workdir)
 
 
-class SourceCodeEventHandler(TriggerlessEventHandler):
+class WebideEventHandler(TriggerlessEventHandler, MonitorManagerMixin):
-    def __init__(self, key, directory, selected_file=None, exclude=None):
+    # pylint: disable=too-many-arguments
+    def __init__(self, key, directory, allowed_directories, selected_file=None, exclude=None):
         super().__init__(key)
-        self.filemanager = FileManager(directory, selected_file=selected_file, exclude=exclude)
+        self.filemanager = FileManager(allowed_directories=allowed_directories, working_directory=directory,
+                                       selected_file=selected_file, exclude=exclude)
+        MonitorManagerMixin.__init__(self, DirectoryMonitor, self.filemanager.workdir)
 
         self.commands = {'read':      self.read,
                          'write':     self.write,
@@ -82,22 +105,6 @@ class SourceCodeEventHandler(TriggerlessEventHandler):
                          'selectdir': self.select_dir,
                          'exclude':   self.exclude}
 
-        self._monitor = None
-        self.reload_monitor()
-
-    @property
-    def monitor(self):
-        return self._monitor
-
-    def reload_monitor(self):
-        if self._monitor:
-            try:
-                self._monitor.stop()
-            except KeyError:
-                logging.debug('Working directory was removed – ignoring...')
-        self._monitor = DirectoryMonitor(self.filemanager.workdir)
-        self._monitor.watch()   # This runs on a separate thread
-
     def read(self, data):
         try:
             data['content'] = self.filemanager.file_contents
@@ -134,8 +141,8 @@ class SourceCodeEventHandler(TriggerlessEventHandler):
                 self.read(data)
             except IndexError:
                 data['content'] = 'No files in this directory :('
-        except EnvironmentError:
+        except EnvironmentError as err:
-            LOG.error('Failed to select directory "%s"', data['directory'])
+            LOG.error('Failed to select directory "%s". Reason: %s', data['directory'], str(err))
         return data
 
     def exclude(self, data):
@@ -150,7 +157,7 @@ class SourceCodeEventHandler(TriggerlessEventHandler):
         data['files'] = self.filemanager.files
         data['directory'] = self.filemanager.workdir
 
-    def handle_event(self, key, message):
+    def handle_event(self, message):
         try:
             data = message['data']
             message['data'] = self.commands[data['command']](data)

lib/tfw/config/__init__.py

@@ -1 +1,4 @@
-from .envvars import *
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from .envvars import TFWENV

lib/tfw/config/envvars.py

@@ -1,3 +1,6 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from envvars import prefixed_envvars_to_namedtuple
 
 TFWENV = prefixed_envvars_to_namedtuple('TFW_', 'tfwenvtuple')

lib/tfw/config/logs.py

@@ -1,3 +1,6 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 import logging
 
 logging.basicConfig(level=logging.DEBUG)

lib/tfw/decorators/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from .rate_limiter import RateLimiter

lib/tfw/decorators/rate_limiter.py

@@ -1,3 +1,6 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from functools import wraps
 from time import time, sleep
 

lib/tfw/event_handler_base.py

@@ -1,7 +1,10 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from abc import ABC, abstractmethod
 
-from tfw.networking.serialization import deserialize_all
+from tfw.networking import deserialize_tfw_msg
-from tfw.networking.event_handlers.server_connector import ServerConnector
+from tfw.networking.event_handlers import ServerConnector
 
 
 class EventHandlerBase(ABC):
@@ -14,25 +17,26 @@ class EventHandlerBase(ABC):
         self.server_connector.register_callback(self.event_handler_callback)
 
     def event_handler_callback(self, msg_parts):
-        key, message = deserialize_all(*msg_parts)
+        message = deserialize_tfw_msg(*msg_parts)
-        response = self.dispatch_handling(key, message)
+        response = self.dispatch_handling(message)
+        response['key'] = message['key']
         if response is None:
             return
-        self.server_connector.send(key, response)
+        self.server_connector.send(response)
 
     @abstractmethod
-    def dispatch_handling(self, key, message):
+    def dispatch_handling(self, message):
         raise NotImplementedError
 
-    def _dispatch_handling(self, key, message):
+    def _dispatch_handling(self, message):
         # pylint: disable=no-else-return
-        if key != 'reset':
+        if message['key'] != 'reset':
-            return self.handle_event(key, message)
+            return self.handle_event(message)
         else:
             return self.handle_reset(message)
 
     @abstractmethod
-    def handle_event(self, key, message):
+    def handle_event(self, message):
         raise NotImplementedError
 
     def handle_reset(self, message):
@@ -42,13 +46,6 @@ class EventHandlerBase(ABC):
     def cleanup(self):
         pass
 
-    def message_other(self, key, data):
-        message = {
-            'key': key,
-            'data': data
-        }
-        self.server_connector.send(key, message)
-
     def subscribe(self, key):
         if key not in self.subscriptions:
             self.subscriptions.add(key)
@@ -69,8 +66,8 @@ class EventHandlerBase(ABC):
 
 class TriggerlessEventHandler(EventHandlerBase, ABC):
     # pylint: disable=abstract-method
-    def dispatch_handling(self, key, message):
+    def dispatch_handling(self, message):
-        return self._dispatch_handling(key, message)
+        return self._dispatch_handling(message)
 
 
 class TriggeredEventHandler(EventHandlerBase, ABC):
@@ -79,7 +76,7 @@ class TriggeredEventHandler(EventHandlerBase, ABC):
         super().__init__(key)
         self.trigger = trigger
 
-    def dispatch_handling(self, key, message):
+    def dispatch_handling(self, message):
         if message.get('trigger') == self.trigger:
-            return self._dispatch_handling(key, message)
+            return self._dispatch_handling(message)
         return None

lib/tfw/fsm_base.py

@@ -1,8 +1,11 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from typing import List
 
 from transitions import Machine
 
-from tfw.components.mixins.callback_mixin import CallbackMixin
+from tfw.mixins import CallbackMixin
 
 
 class FSMBase(CallbackMixin):

lib/tfw/linear_fsm.py

@@ -0,0 +1,12 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from .fsm_base import FSMBase
+
+
+class LinearFSM(FSMBase):
+    def __init__(self, number_of_steps):
+        self.states = list(map(str, range(number_of_steps)))
+        self.transitions = [{'trigger': 'step_{}'.format(int(index)+1), 'source': index, 'dest': str(int(index)+1)}
+                            for index in self.states[:-1]]
+        super(LinearFSM, self).__init__()

lib/tfw/mixins/__init__.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from .supervisor_mixin import SupervisorMixin
+from .callback_mixin import CallbackMixin
+from .observer_mixin import ObserverMixin
+from .monitor_manager_mixin import MonitorManagerMixin

lib/tfw/mixins/callback_mixin.py

@@ -1,3 +1,6 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from functools import partial
 
 

lib/tfw/mixins/monitor_manager_mixin.py

@@ -0,0 +1,27 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from tfw.config.logs import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class MonitorManagerMixin:
+    def __init__(self, monitor_type, directory):
+        self._monitor_type = monitor_type
+        self._monitor = None
+        self._monitored_directory = directory
+        self.reload_monitor()
+
+    @property
+    def monitor(self):
+        return self._monitor
+
+    def reload_monitor(self):
+        if self._monitor:
+            try:
+                self._monitor.stop()
+            except KeyError:
+                LOG.debug('Working directory was removed – ignoring...')
+        self._monitor = self._monitor_type(self._monitored_directory)
+        self._monitor.watch()   # This runs on a separate thread

lib/tfw/mixins/observer_mixin.py

@@ -0,0 +1,16 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from watchdog.observers import Observer
+
+
+class ObserverMixin:
+    def __init__(self):
+        self.observer = Observer()
+
+    def watch(self):
+        self.observer.start()
+
+    def stop(self):
+        self.observer.stop()
+        self.observer.join()

lib/tfw/mixins/supervisor_mixin.py

@@ -1,3 +1,6 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 import xmlrpc.client
 from xmlrpc.client import Fault as SupervisorFault
 from contextlib import suppress

lib/tfw/networking/__init__.py

@@ -0,0 +1,9 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from .serialization import serialize_tfw_msg, deserialize_tfw_msg, validate_message
+from .zmq_connector_base import ZMQConnectorBase
+# from .controller_connector import ControllerConnector # TODO: readd once controller stuff is resolved
+from .message_sender import MessageSender
+from .event_handlers.server_connector import ServerUplinkConnector as TFWServerConnector
+from .server.tfw_server import TFWServer

lib/tfw/networking/controller_connector.py

@@ -1,8 +1,11 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 import zmq
 from zmq.eventloop.zmqstream import ZMQStream
 
 from tfw.config import TFWENV
-from tfw.networking.zmq_connector_base import ZMQConnectorBase
+from tfw.networking import ZMQConnectorBase
 
 
 class ControllerConnector(ZMQConnectorBase):

lib/tfw/networking/event_handlers/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from .server_connector import ServerConnector, ServerUplinkConnector, ServerDownlinkConnector

lib/tfw/networking/event_handlers/server_connector.py

@@ -1,11 +1,14 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from functools import partial
 
 import zmq
 from zmq.eventloop.zmqstream import ZMQStream
 
-from tfw.networking.serialization import serialize_all
+from tfw.networking import serialize_tfw_msg
+from tfw.networking import ZMQConnectorBase
 from tfw.config import TFWENV
-from tfw.networking.zmq_connector_base import ZMQConnectorBase
 
 
 class ServerDownlinkConnector(ZMQConnectorBase):
@@ -26,13 +29,14 @@ class ServerUplinkConnector(ZMQConnectorBase):
         self._zmq_push_socket = self._zmq_context.socket(zmq.PUSH)
         self._zmq_push_socket.connect('tcp://localhost:{}'.format(TFWENV.RECEIVER_PORT))
 
-    def send_to_eventhandler(self, key, message):
+    def send_to_eventhandler(self, message):
-        message['data']['key'] = key
+        nested_message = {'key': message['key'], 'data': message.pop('data')}
-        self.send('mirror', message)
+        message['key'] = 'mirror'
+        message['data'] = nested_message
+        self.send(message)
 
-    def send(self, key, message):
+    def send(self, message):
-        message['key'] = key
+        self._zmq_push_socket.send_multipart(serialize_tfw_msg(message))
-        self._zmq_push_socket.send_multipart(serialize_all(key, message))
 
 
 class ServerConnector(ServerUplinkConnector, ServerDownlinkConnector):

lib/tfw/networking/message_sender.py

@@ -1,6 +1,9 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from datetime import datetime
 
-from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
+from tfw.networking.event_handlers import ServerUplinkConnector
 
 
 class MessageSender:
@@ -14,5 +17,5 @@ class MessageSender:
             'timestamp': datetime.now().isoformat(),
             'message': message
         }
-        response = {'data': data}
+        self.server_connector.send({'key': self.key,
-        self.server_connector.send(self.key, response)
+                                    'data': data})

lib/tfw/networking/serialization.py

@@ -1,21 +1,69 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+"""
+TFW JSON message format
+
+message:
+{
+    "key":     string,    # addressing
+    "data":    {...},     # payload
+    "trigger": string     # FSM trigger
+}
+
+ZeroMQ's sub-pub sockets use enveloped messages
+(http://zguide.zeromq.org/page:all#Pub-Sub-Message-Envelopes)
+and TFW also uses them internally. This means that on ZMQ sockets
+we always send the messages key separately and then the actual
+message (which contains the key as well) like so:
+
+socket.send_multipart([message['key'], message])
+
+The purpose of this module is abstracting away this low level behaviour.
+"""
+
 import json
 
 
-def encode_if_needed(value):
+def validate_message(message):
+    return 'key' in message
+
+
+def serialize_tfw_msg(message):
+    return _serialize_all(message['key'], message)
+
+
+def deserialize_tfw_msg(*args):
+    return _deserialize_all(*args)[1]
+
+
+def _serialize_all(*args):
+    return tuple(_serialize_single(arg) for arg in args)
+
+
+def _deserialize_all(*args):
+    return tuple(_deserialize_single(arg) for arg in args)
+
+
+def _serialize_single(data):
+    if not isinstance(data, str):
+        data = json.dumps(data)
+    return _encode_if_needed(data)
+
+
+def _deserialize_single(data):
+    try:
+        return json.loads(data)
+    except ValueError:
+        return _decode_if_needed(data)
+
+
+def _encode_if_needed(value):
     if isinstance(value, str):
         value = value.encode('utf-8')
     return value
 
 
-def decode_if_needed(value):
+def _decode_if_needed(value):
     if isinstance(value, (bytes, bytearray)):
         value = value.decode('utf-8')
     return value
-
-
-def serialize_all(key, data):
-    return [encode_if_needed(frame) for frame in (key, json.dumps(data))]
-
-
-def deserialize_all(key, data):
-    return decode_if_needed(key), json.loads(data)

lib/tfw/networking/server/__init__.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
+from .event_handler_connector import EventHandlerConnector, EventHandlerUplinkConnector, EventHandlerDownlinkConnector
+from .tfw_server import TFWServer
+from .zmq_websocket_handler import ZMQWebSocketProxy
+# from .controller_responder import ControllerResponder  # TODO: readd once controller stuff is resolved

lib/tfw/networking/server/controller_responder.py

@@ -1,5 +1,7 @@
-from tfw.networking.controller_connector import ControllerConnector
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
-from tfw.networking.serialization import deserialize_all, serialize_all
+# All Rights Reserved. See LICENSE file for details.
+
+from tfw.networking import deserialize_all, serialize_all, ControllerConnector
 
 
 class ControllerResponder:

lib/tfw/networking/server/event_handler_connector.py

@@ -1,8 +1,10 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 import zmq
 from zmq.eventloop.zmqstream import ZMQStream
 
-from tfw.networking.zmq_connector_base import ZMQConnectorBase
+from tfw.networking import ZMQConnectorBase, serialize_tfw_msg
-from tfw.networking.serialization import serialize_all
 from tfw.config import TFWENV
 from tfw.config.logs import logging
 
@@ -32,7 +34,5 @@ class EventHandlerConnector(EventHandlerDownlinkConnector, EventHandlerUplinkCon
     def register_callback(self, callback):
         self._zmq_pull_stream.on_recv(callback)
 
-    def send_message(self, message: dict, key: str = None):
+    def send_message(self, message: dict):
-        if not key:
+        self._zmq_pub_socket.send_multipart(serialize_tfw_msg(message))
-            key = message.get('key', '')
-        self._zmq_pub_socket.send_multipart(serialize_all(key, message))

lib/tfw/networking/server/tfw_server.py

@@ -1,11 +1,15 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 from collections import defaultdict
 
 from tornado.web import Application
 
-from tfw.networking.server.zmq_websocket_handler import ZMQWebSocketProxy
+from tfw.networking import MessageSender
-from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
+from tfw.networking.event_handlers import ServerUplinkConnector
-from tfw.message_sender import MessageSender
+from tfw.networking.server import EventHandlerConnector
 from tfw.config.logs import logging
+from .zmq_websocket_handler import ZMQWebSocketProxy
 
 LOG = logging.getLogger(__name__)
 
@@ -16,10 +20,13 @@ class TFWServer:
         self._fsm_updater = FSMUpdater(self._fsm)
         self._fsm_manager = FSMManager(self._fsm)
         self._fsm.subscribe_callback(self._fsm_updater.update)
+        self._event_handler_connector = EventHandlerConnector()
 
         self.application = Application(
-            [(r'/ws', ZMQWebSocketProxy, {'make_response': self.make_response,
+            [(r'/ws', ZMQWebSocketProxy, {'make_eventhandler_message': self.make_eventhandler_message,
-                                          'proxy_filter': self.proxy_filter})]
+                                          'proxy_filter': self.proxy_filter,
+                                          'handle_trigger': self.handle_trigger,
+                                          'event_handler_connector': self._event_handler_connector})]
         )
         #self.controller_responder = ControllerResponder(self.fsm) TODO: add this once controller stuff is resolved
 
@@ -31,13 +38,21 @@ class TFWServer:
     def fsm_manager(self):
         return self._fsm_manager
 
-    def make_response(self, message):
+    def make_eventhandler_message(self, message):
+        self.trigger_fsm(message)
+        message['FSMUpdate'] = self._fsm_updater.get_fsm_state_and_transitions()
+        return message
+
+    def handle_trigger(self, message):
+        LOG.debug('Executing handler for trigger "%s"', message.get('trigger', ''))
+        self.trigger_fsm(message)
+
+    def trigger_fsm(self, message):
         trigger = message.get('trigger', '')
         try:
             self._fsm_manager.trigger(trigger, message)
         except AttributeError:
             LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
-        return message
 
     def proxy_filter(self, message):
         # pylint: disable=unused-argument,no-self-use
@@ -60,7 +75,7 @@ class FSMManager:
     def trigger(self, trigger, message):
         predicate_results = []
         for predicate in self.trigger_predicates[trigger]:
-            success, message = predicate()
+            success, message = predicate(message)
             predicate_results.append(success)
             self.messenge_sender.send('FSM', message)
 
@@ -85,11 +100,13 @@ class FSMUpdater:
 
     def update(self, kwargs_dict):
         # pylint: disable=unused-argument
-        self.uplink.send(*self.generate_fsm_update())
+        self.uplink.send(self.generate_fsm_update())
 
     def generate_fsm_update(self):
-        key = 'FSMUpdate'
+        return {'key': 'FSMUpdate',
-        response = {'data': {'current_state': self.fsm.state,
+                'data': self.get_fsm_state_and_transitions()}
-                             'valid_transitions':
+
-                                 [{'trigger': trigger} for trigger in self.fsm.machine.get_triggers(self.fsm.state)]}}
+    def get_fsm_state_and_transitions(self):
-        return key, response
+        state = self.fsm.state
+        valid_transitions = [{'trigger': trigger} for trigger in self.fsm.machine.get_triggers(self.fsm.state)]
+        return {'current_state': state, 'valid_transitions': valid_transitions}

lib/tfw/networking/server/zmq_websocket_handler.py

@@ -1,19 +1,23 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 import json
 from abc import ABC, abstractmethod
 
 from tornado.websocket import WebSocketHandler
 
-from tfw.networking.serialization import deserialize_all
+from tfw.networking import deserialize_tfw_msg, validate_message
-from tfw.networking.server.event_handler_connector import EventHandlerConnector
 from tfw.config.logs import logging
 
 LOG = logging.getLogger(__name__)
 
 
 class ZMQWebSocketHandler(WebSocketHandler, ABC):
-    _event_handler_connector = EventHandlerConnector()
     instances = set()
 
+    def initialize(self, **kwargs): # pylint: disable=arguments-differ
+        self._event_handler_connector = kwargs['event_handler_connector']
+
     def prepare(self):
         ZMQWebSocketHandler.instances.add(self)
 
@@ -24,36 +28,43 @@ class ZMQWebSocketHandler(WebSocketHandler, ABC):
         LOG.debug('WebSocket connection initiated')
         self._event_handler_connector.register_callback(self.zmq_callback)
 
-    @classmethod
+    def zmq_callback(self, msg_parts):
-    def zmq_callback(cls, msg_parts):
+        keyhandlers = {'mirror': self.mirror}
-        keyhandlers = {'mirror': cls.mirror}
 
-        key, data = deserialize_all(*msg_parts)
+        message = deserialize_tfw_msg(*msg_parts)
-        LOG.debug('Received on pull socket: %s', data)
+        LOG.debug('Received on pull socket: %s', message)
-        if key not in keyhandlers:
+        if not validate_message(message):
-            for instance in cls.instances:
+            return
-                instance.write_message(data)
+
+        self.handle_trigger(message)
+        if message['key'] not in keyhandlers:
+            for instance in ZMQWebSocketHandler.instances:
+                instance.write_message(message)
         else:
             try:
-                keyhandlers[key](data['data'])
+                keyhandlers[message['key']](message)
             except KeyError:
                 LOG.error('Invalid mirror message format! Ignoring.')
 
-    @classmethod
+    def mirror(self, message):
-    def mirror(cls, data):
+        message = message['data']
-        key = data['key']
+        self._event_handler_connector.send_message(message)
-        cls._event_handler_connector.send_message({'data': data}, key)
 
     def on_message(self, message):
         LOG.debug('Received on WebSocket: %s', message)
-        self.send_message(self.make_response(message))
+        if validate_message(message):
+            self.send_message(self.make_eventhandler_message(message))
 
     @abstractmethod
-    def make_response(self, message):
+    def make_eventhandler_message(self, message):
         raise NotImplementedError
 
-    def send_message(self, message: dict, key: str = None):
+    def send_message(self, message: dict):
-        self._event_handler_connector.send_message(message, key)
+        self._event_handler_connector.send_message(message)
+
+    @abstractmethod
+    def handle_trigger(self, message):
+        raise NotImplementedError
 
     # much secure, very cors, wow
     def check_origin(self, origin):
@@ -62,15 +73,19 @@ class ZMQWebSocketHandler(WebSocketHandler, ABC):
 
 class ZMQWebSocketProxy(ZMQWebSocketHandler):
     # pylint: disable=abstract-method
-    def initialize(self, make_response, proxy_filter):
+    def initialize(self, **kwargs): # pylint: disable=arguments-differ
-        # pylint: disable=arguments-differ
+        super(ZMQWebSocketProxy, self).initialize(**kwargs)
-        self._make_response = make_response
+        self._make_eventhandler_message = kwargs['make_eventhandler_message']
-        self._proxy_filter = proxy_filter
+        self._proxy_filter = kwargs['proxy_filter']
+        self._handle_trigger = kwargs['handle_trigger']
 
     def on_message(self, message):
         message = json.loads(message)
         if self._proxy_filter(message):
             super().on_message(message)
 
-    def make_response(self, message):
+    def make_eventhandler_message(self, message):
-        return self._make_response(message)
+        return self._make_eventhandler_message(message)
+
+    def handle_trigger(self, message):
+        self._handle_trigger(message)

lib/tfw/networking/zmq_connector_base.py

@@ -1,3 +1,6 @@
+# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
+# All Rights Reserved. See LICENSE file for details.
+
 import zmq
 
 

supervisor/supervisord.conf

@@ -1,5 +1,5 @@
 [supervisord]
-user=user
+user=root
 logfile = /tmp/supervisord.log
 loglevel = debug
 pidfile = /tmp/supervisord.pid
@@ -18,9 +18,5 @@ command=/usr/sbin/nginx -g 'daemon off;'
 autostart=true
 autorestart=true
 
-[program:app]
-directory=%(ENV_TFW_APP_DIR)s
-command=python3 app.py
-
 [include]
 files=%(ENV_TFW_SUPERVISORD_COMPONENTS)s/*.conf