Document TFW message format in README

Dockerfile

@@ -39,20 +39,15 @@ ENV PYTHONPATH="/usr/local/lib/"                                           \
     TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history"                      \
     PROMPT_COMMAND="history -a"
 
-RUN echo "shopt -s cmdhist\n"                 \
+COPY bashrc /tmp
-         "shopt -s histappend\n"              \
+RUN echo "export HISTFILE=${TFW_HISTFILE}\n" >> /tmp/bashrc &&\
-         "unset HISTCONTROL\n"                \
+    cat /tmp/bashrc >> /home/${AVATAO_USER}/.bashrc
-         "export HISTFILE=$TFW_HISTFILE\n"    \
-         "export HISTFILESIZE=1000\n"         \
-         "export HISTSIZE=1000\n"             \
-         'PROMPT_COMMAND="history -a"\n'      \
-    >> /home/${AVATAO_USER}/.bashrc
 
 COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF}
 COPY nginx/nginx.conf ${TFW_NGINX_CONF}
 COPY nginx/default.conf ${TFW_NGINX_DEFAULT}
 COPY nginx/components/ ${TFW_NGINX_COMPONENTS}
-COPY lib ${TFW_LIB_DIR}
+COPY lib LICENSE ${TFW_LIB_DIR}
 
 RUN for dir in "${TFW_LIB_DIR}" "/etc/nginx" "/etc/supervisor"; do \
         chown -R root:root "$dir" && chmod -R 700 "$dir";          \

README.md

@@ -50,6 +50,32 @@ This includes context-dependent hints and the automatic typing of commands to a
 
 Note that our frontend implementation is written in Angular. It is maintained and documented in the `frontend-tutorial-framework` repository.
 
+### Messaging format
+
+The framework uses JSON messages internally and in exposed APIs as well.
+These messages must comply some rules.
+Don't worry, we are not too fond of rules around these parts.
+
+The TFW message format:
+
+```json
+{
+    "key: "some identifier used for addressing",
+    "data":
+    {
+        ...
+        JSON object carrying anything, preferably cats
+        ...
+    },
+    "trigger": "FSM action"
+}
+```
+
+- The `key` field is used by TFW for addressing and every message must have one (it can be an empty string though)
+- The `data` object can contain anything you might want to send
+- The `trigger` key is an optional field that triggers an FSM action with that name from the current state (whatever that might be)
+
+
 ## Where to go next
 
 Most of the components you need have docstrings included (hang on tight, this is work in progress) – refer to them for usage info.

bashrc

@@ -0,0 +1,20 @@
+_postcmd() 
+{
+    AT_PROMPT=1
+    history -a
+}
+PROMPT_COMMAND="_postcmd"
+
+_precmd()
+{
+    [ -z "$AT_PROMPT" ] && return
+    unset AT_PROMPT
+    history -a
+}
+trap _precmd DEBUG
+
+shopt -s cmdhist
+shopt -s histappend
+unset HISTCONTROL
+export HISTFILESIZE=1000
+export HISTSIZE=1000

lib/tfw/__init__.py

@@ -1,6 +1,6 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from .event_handler_base import TriggerlessEventHandler, TriggeredEventHandler
+from .event_handler_base import EventHandlerBase, TriggeredEventHandler
 from .fsm_base import FSMBase
 from .linear_fsm import LinearFSM

lib/tfw/components/directory_monitoring_event_handler.py

@@ -3,7 +3,7 @@
 
 from os.path import isdir, exists
 
-from tfw import TriggerlessEventHandler
+from tfw import EventHandlerBase
 from tfw.config.logs import logging
 from tfw.mixins import MonitorManagerMixin
 from .directory_monitor import DirectoryMonitor
@@ -11,7 +11,7 @@ from .directory_monitor import DirectoryMonitor
 LOG = logging.getLogger(__name__)
 
 
-class DirectoryMonitoringEventHandler(TriggerlessEventHandler, MonitorManagerMixin):
+class DirectoryMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin):
     def __init__(self, key, directory):
         super().__init__(key)
         self._directory = directory

lib/tfw/components/history_monitor.py

@@ -24,6 +24,17 @@ class CallbackEventHandler(PatternMatchingEventHandler, ABC):
 
 
 class HistoryMonitor(CallbackMixin, ObserverMixin, ABC):
+    """
+    Abstract class capable of monitoring and parsing a history file such as
+    bash HISTFILEs. Monitoring means detecting when the file was changed and
+    notifying subscribers about new content in the file.
+
+    This is useful for monitoring CLI sessions.
+
+    To specify a custom HistoryMonitor inherit from this class and override the
+    command pattern property and optionally the sanitize_command method.
+    See examples below.
+    """
     def __init__(self, histfile):
         CallbackMixin.__init__(self)
         ObserverMixin.__init__(self)
@@ -61,6 +72,15 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC):
 
 
 class BashMonitor(HistoryMonitor):
+    """
+    HistoryMonitor for monitoring bash CLI sessions.
+    This requires the following to be set in bash
+    (note that this is done automatically by TFW):
+        PROMPT_COMMAND="history -a"
+        shopt -s cmdhist
+        shopt -s histappend
+        unset HISTCONTROL
+    """
     @property
     def command_pattern(self):
         return r'.+'
@@ -70,6 +90,10 @@ class BashMonitor(HistoryMonitor):
 
 
 class GDBMonitor(HistoryMonitor):
+    """
+    HistoryMonitor to monitor GDB sessions.
+    For this to work "set trace-commands on" must be set in GDB.
+    """
     @property
     def command_pattern(self):
         return r'(?<=\n)\+(.+)\n'

lib/tfw/components/process_managing_event_handler.py

@@ -3,7 +3,7 @@
 
 from xmlrpc.client import Fault as SupervisorFault
 
-from tfw import TriggerlessEventHandler
+from tfw import EventHandlerBase
 from tfw.mixins import SupervisorMixin
 from tfw.config.logs import logging
 from .directory_monitor import with_monitor_paused
@@ -22,7 +22,19 @@ class ProcessManager(SupervisorMixin):
         return self.commands[command](process_name)
 
 
-class ProcessManagingEventHandler(TriggerlessEventHandler):
+class ProcessManagingEventHandler(EventHandlerBase):
+    """
+    Event handler that can manage processes managed by supervisor.
+
+    This EventHandler accepts messages that have a data["command"] key specifying
+    a command to be executed.
+    Every message must contain a data["process_name"] field with the name of the
+    process to manage. This is the name specified in supervisor config files like so:
+    [program:someprogram]
+
+    Commands available: start, stop, restart, readlog
+    (the names are as self-documenting as it gets)
+    """
     def __init__(self, key, dirmonitor=None):
         super().__init__(key)
         self.key = key

lib/tfw/components/terminado_event_handler.py

@@ -1,7 +1,7 @@
 # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
 # All Rights Reserved. See LICENSE file for details.
 
-from tfw import TriggerlessEventHandler
+from tfw import EventHandlerBase
 from tfw.config import TFWENV
 from tfw.config.logs import logging
 from tao.config import TAOENV
@@ -10,8 +10,21 @@ from .terminado_mini_server import TerminadoMiniServer
 LOG = logging.getLogger(__name__)
 
 
-class TerminadoEventHandler(TriggerlessEventHandler):
+class TerminadoEventHandler(EventHandlerBase):
+    """
+    Event handler responsible for managing terminal sessions for frontend xterm
+    sessions to connect to. You need to instanciate this in order for frontend
+    terminals to work.
+
+    This EventHandler accepts messages that have a data["command"] key specifying
+    a command to be executed.
+    The API of each command is documented in their respective handlers.
+    """
     def __init__(self, key, monitor):
+        """
+        :param key: key this EventHandler listens to
+        :param monitor: tfw.components.HistoryMonitor instance to read command history from
+        """
         super().__init__(key)
         self.working_directory = TFWENV.TERMINADO_DIR
         self._historymonitor = monitor
@@ -37,9 +50,21 @@ class TerminadoEventHandler(TriggerlessEventHandler):
             LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
 
     def write(self, data):
+        """
+        Writes a string to the terminal session (on the pty level).
+        Useful for pre-typing and executing commands for the user.
+
+        :param data['shellcmd']: command to be written to the pty
+        """
         self.terminado_server.pty.write(data['shellcmd'])
 
     def read(self, data):
+        """
+        Reads the history of commands executed.
+
+        :param data['count']: the number of history elements to return
+        :return: message with list of commands in data['history']
+        """
         data['count'] = int(data.get('count', 1))
         if self.historymonitor:
             data['history'] = self.historymonitor.history[-data['count']:]

lib/tfw/components/terminal_commands.py

@@ -10,6 +10,24 @@ LOG = logging.getLogger(__name__)
 
 
 class TerminalCommands(ABC):
+    """
+    A class you can use to define hooks for terminal commands. This means that you can
+    have python code executed when the user enters a specific command to the terminal on
+    our frontend.
+
+    To receive events you need to subscribe TerminalCommand.callback to a HistoryMonitor
+    instance.
+
+    Inherit from this class and define methods which start with "command_". When the user
+    executes the command specified after the underscore, your method will be invoked. All
+    such commands must expect the parameter *args which will contain the arguments of the
+    command.
+
+    For example to define a method that runs when someone starts vim in the terminal
+    you have to define a method like: "def command_vim(self, *args)"
+
+    You can also use this class to create new commands similarly.
+    """
     def __init__(self, bashrc=None):
         self._command_method_regex = r'^command_(.+)$'
         self.command_implemetations = self._build_command_to_implementation_dict()
@@ -22,7 +40,7 @@ class TerminalCommands(ABC):
 
     def _setup_bashrc_aliases(self, bashrc):
         with open(bashrc, 'a') as ofile:
-            alias_template = 'alias {0}="{0} > /dev/null 2>&1"\n'
+            alias_template = 'type {0} &> /dev/null || alias {0}="{0} &> /dev/null"\n'
             for command in self.command_implemetations.keys():
                 ofile.write(alias_template.format(command))
 

lib/tfw/components/webide_event_handler.py

@@ -6,7 +6,7 @@ from glob import glob
 from fnmatch import fnmatchcase
 from collections import Iterable
 
-from tfw import TriggerlessEventHandler
+from tfw import EventHandlerBase
 from tfw.mixins import MonitorManagerMixin
 from tfw.config.logs import logging
 from .directory_monitor import DirectoryMonitor
@@ -91,9 +91,25 @@ class FileManager:  # pylint: disable=too-many-instance-attributes
         return relpath(self._filepath(filename), start=self._workdir)
 
 
-class WebideEventHandler(TriggerlessEventHandler, MonitorManagerMixin):
+class WebideEventHandler(EventHandlerBase, MonitorManagerMixin):
     # pylint: disable=too-many-arguments
+    """
+    Event handler implementing the backend of our browser based IDE.
+    By default all files in the directory specified in __init__ are displayed
+    on the fontend. Note that this is a stateful component.
+
+    This EventHandler accepts messages that have a data["command"] key specifying
+    a command to be executed.
+    The API of each command is documented in their respective handlers.
+    """
     def __init__(self, key, directory, allowed_directories, selected_file=None, exclude=None):
+        """
+        :param key: the key this instance should listen to
+        :param directory: working directory which the EventHandler should serve files from
+        :param allowed_directories: list of directories that can be switched to using the selectdir command
+        :param selected_file: file that is selected by default
+        :param exclude: list of filenames that should not appear between files (for *.o, *.pyc, etc.)
+        """
         super().__init__(key)
         self.filemanager = FileManager(allowed_directories=allowed_directories, working_directory=directory,
                                        selected_file=selected_file, exclude=exclude)
@@ -106,6 +122,11 @@ class WebideEventHandler(TriggerlessEventHandler, MonitorManagerMixin):
                          'exclude':   self.exclude}
 
     def read(self, data):
+        """
+        Read the currently selected file.
+
+        :return: message with the contents of the file in data['content']
+        """
         try:
             data['content'] = self.filemanager.file_contents
         except PermissionError:
@@ -117,6 +138,11 @@ class WebideEventHandler(TriggerlessEventHandler, MonitorManagerMixin):
         return data
 
     def write(self, data):
+        """
+        Overwrites a file with the desired string.
+
+        :param data['content']: string containing the desired file contents
+        """
         self.monitor.ignore = self.monitor.ignore + 1
         try:
             self.filemanager.file_contents = data['content']
@@ -126,6 +152,11 @@ class WebideEventHandler(TriggerlessEventHandler, MonitorManagerMixin):
         return data
 
     def select(self, data):
+        """
+        Selects a file from the current directory.
+
+        :param data['filename']: name of file to select relative to the current directory
+        """
         try:
             self.filemanager.filename = data['filename']
         except EnvironmentError:
@@ -133,6 +164,13 @@ class WebideEventHandler(TriggerlessEventHandler, MonitorManagerMixin):
         return data
 
     def select_dir(self, data):
+        """
+        Select a new working directory to display files from.
+
+        :param data['directory']: absolute path of diretory to select.
+                                  must be a path whitelisted in
+                                  self.allowed_directories
+        """
         try:
             self.filemanager.workdir = data['directory']
             self.reload_monitor()
@@ -146,6 +184,11 @@ class WebideEventHandler(TriggerlessEventHandler, MonitorManagerMixin):
         return data
 
     def exclude(self, data):
+        """
+        Overwrite list of excluded files
+
+        :param data['exclude']: list of filename patterns to be excluded, e.g.: ["*.pyc", "*.o"]
+        """
         try:
             self.filemanager.exclude = list(data['exclude'])
         except TypeError:
@@ -153,6 +196,9 @@ class WebideEventHandler(TriggerlessEventHandler, MonitorManagerMixin):
         return data
 
     def attach_fileinfo(self, data):
+        """
+        Basic information included in every response to the frontend.
+        """
         data['filename'] = self.filemanager.filename
         data['files'] = self.filemanager.files
         data['directory'] = self.filemanager.workdir

lib/tfw/event_handler_base.py

@@ -8,75 +8,97 @@ from tfw.networking.event_handlers import ServerConnector
 
 
 class EventHandlerBase(ABC):
+    """
+    Abstract base class for all Python based EventHandlers. Useful implementation template
+    for other languages.
+
+    Derived classes must implement the handle_event() method
+    """
     def __init__(self, key):
         self.server_connector = ServerConnector()
         self.key = key
-        self.subscriptions = set()
+        self.subscribe(self.key, 'reset')
-        self.subscribe(self.key)
-        self.subscribe('reset')
         self.server_connector.register_callback(self.event_handler_callback)
 
     def event_handler_callback(self, msg_parts):
+        """
+        Callback that is invoked when receiving a message.
+        Dispatches messages to handler methods and sends
+        a response back in case the handler returned something.
+        This is subscribed in __init__().
+        """
         message = deserialize_tfw_msg(*msg_parts)
         response = self.dispatch_handling(message)
-        response['key'] = message['key']
+        if response:
-        if response is None:
+            response['key'] = message['key']
-            return
+            self.server_connector.send(response)
-        self.server_connector.send(response)
 
-    @abstractmethod
     def dispatch_handling(self, message):
-        raise NotImplementedError
+        """
-
+        Used to dispatch messages to their specific handlers.
-    def _dispatch_handling(self, message):
+        """
-        # pylint: disable=no-else-return
         if message['key'] != 'reset':
             return self.handle_event(message)
-        else:
+        return self.handle_reset(message)
-            return self.handle_reset(message)
 
     @abstractmethod
     def handle_event(self, message):
+        """
+        Abstract method that implements the handling of messages.
+
+        :param message: the message received
+        """
         raise NotImplementedError
 
     def handle_reset(self, message):
         # pylint: disable=unused-argument,no-self-use
+        """
+        Usually 'reset' events receive some sort of special treatment.
+
+        :param message: the message received
+        """
         return None
 
-    def cleanup(self):
+    def subscribe(self, *keys):
-        pass
+        """
-
+        Subscribe this EventHandler to receive events for given keys.
-    def subscribe(self, key):
+        Note that you can subscribe to the same key several times in which
-        if key not in self.subscriptions:
+        case you will need to unsubscribe multiple times in order to stop
-            self.subscriptions.add(key)
+        receiving events.
+        
+        :param keys: list of keys to subscribe to
+        """
+        for key in keys:
             self.server_connector.subscribe(key)
 
-    def unsubscribe(self, key):
+    def unsubscribe(self, *keys):
-        try:
+        """
-            self.subscriptions.remove(key)
+        Unsubscribe this eventhandler from the given keys.
+
+        :param keys: list of keys to unsubscribe from
+        """
+        for key in keys:
             self.server_connector.unsubscribe(key)
-        except KeyError:
-            pass
 
-    def unsubscribe_all(self):
+    def cleanup(self):
-        for sub in self.subscriptions:
+        """
-            self.server_connector.unsubscribe(key=sub)
+        Perform cleanup actions such as releasing database
-        self.subscriptions.clear()
+        connections and stuff like that.
-
+        """
-
+        pass
-class TriggerlessEventHandler(EventHandlerBase, ABC):
-    # pylint: disable=abstract-method
-    def dispatch_handling(self, message):
-        return self._dispatch_handling(message)
 
 
 class TriggeredEventHandler(EventHandlerBase, ABC):
     # pylint: disable=abstract-method
+    """
+    Abstract base class for EventHandlers which are only triggered in case
+    TFWServer has successfully triggered an FSM step defined in __init__.
+    """
     def __init__(self, key, trigger):
         super().__init__(key)
         self.trigger = trigger
 
     def dispatch_handling(self, message):
         if message.get('trigger') == self.trigger:
-            return self._dispatch_handling(message)
+            return super().dispatch_handling(message)
         return None

lib/tfw/fsm_base.py

@@ -9,6 +9,13 @@ from tfw.mixins import CallbackMixin
 
 
 class FSMBase(CallbackMixin):
+    """
+    A general FSM base class you can inherit from to track user progress.
+    See linear_fsm.py for an example use-case.
+    TFW the transitions library for state machines, please refer to their
+    documentation for more information on creating your own machines:
+    https://github.com/pytransitions/transitions
+    """
     states, transitions = [], []
 
     def __init__(self, initial: str = None, accepted_states: List[str] = None):

lib/tfw/linear_fsm.py

@@ -5,6 +5,12 @@ from .fsm_base import FSMBase
 
 
 class LinearFSM(FSMBase):
+    """
+    This is a state machine for challenges with linear progression, consisting of
+    a number of steps specified in the constructor. It automatically sets up a single
+    action between states as such:
+    0 ==step_1==> 1 ==step_2==> 2 ==step_3==> 3 ... and so on
+    """
     def __init__(self, number_of_steps):
         self.states = list(map(str, range(number_of_steps)))
         self.transitions = [{'trigger': 'step_{}'.format(int(index)+1), 'source': index, 'dest': str(int(index)+1)}

lib/tfw/mixins/callback_mixin.py

@@ -9,6 +9,12 @@ class CallbackMixin:
         self._callbacks = []
 
     def subscribe_callback(self, callback, *args, **kwargs):
+        """
+        Subscribe a callable to invoke once an event is triggered.
+        :param callback: callable to be executed on events
+        :param *args: arguments passed to callable
+        :param **kwargs: kwargs passed to callable
+        """
         fun = partial(callback, *args, **kwargs)
         self._callbacks.append(fun)
 

lib/tfw/networking/event_handlers/server_connector.py

@@ -24,18 +24,30 @@ class ServerDownlinkConnector(ZMQConnectorBase):
 
 
 class ServerUplinkConnector(ZMQConnectorBase):
+    """
+    Class capable of sending messages to the TFW server and event handlers.
+    """
     def __init__(self, zmq_context=None):
         super(ServerUplinkConnector, self).__init__(zmq_context)
         self._zmq_push_socket = self._zmq_context.socket(zmq.PUSH)
         self._zmq_push_socket.connect('tcp://localhost:{}'.format(TFWENV.RECEIVER_PORT))
 
     def send_to_eventhandler(self, message):
+        """
+        Send a message to an event handler.
+        :param message: JSON message you want to send
+        :param message['key']: key of event handler you want to address
+        """
         nested_message = {'key': message['key'], 'data': message.pop('data')}
         message['key'] = 'mirror'
         message['data'] = nested_message
         self.send(message)
 
     def send(self, message):
+        """
+        Send a message to the TFW server
+        :param message: JSON message you want to send
+        """
         self._zmq_push_socket.send_multipart(serialize_tfw_msg(message))
 
 

lib/tfw/networking/message_sender.py

@@ -7,11 +7,20 @@ from tfw.networking.event_handlers import ServerUplinkConnector
 
 
 class MessageSender:
+    """
+    Provides a mechanism to send messages to our frontend messaging component which
+    displays messages with the key "message".
+    """
     def __init__(self, custom_key: str = None):
         self.server_connector = ServerUplinkConnector()
         self.key = custom_key or 'message'
 
     def send(self, originator, message):
+        """
+        Sends a message to the key specified in __init__.
+        :param originator: name of sender to be displayed on the frontend
+        :param message: message to send
+        """
         data = {
             'originator': originator,
             'timestamp': datetime.now().isoformat(),

lib/tfw/networking/server/tfw_server.py

@@ -15,7 +15,15 @@ LOG = logging.getLogger(__name__)
 
 
 class TFWServer:
+    """
+    This class handles the proxying of messages between the frontend and event handers.
+    It proxies messages from the "/ws" route to all event handlers subscribed to a ZMQ
+    SUB socket. It also manages an FSM you can define as a constructor argument.
+    """
     def __init__(self, fsm_type):
+        """
+        :param fsm_type: the type of FSM you want TFW to use
+        """
         self._fsm = fsm_type()
         self._fsm_updater = FSMUpdater(self._fsm)
         self._fsm_manager = FSMManager(self._fsm)