Merge pull request #33 from avatao-content/message-types

Message types
This commit is contained in:
therealkrispet 2018-06-04 22:18:25 +02:00 committed by GitHub
commit ab966f6d27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 645 additions and 153 deletions

View File

@ -10,4 +10,4 @@ pipeline:
- docker push eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG} - docker push eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG}
when: when:
event: 'tag' event: 'tag'
branch: refs/tags/egyptianmau-20* branch: refs/tags/bombay-20*

View File

@ -1,5 +1,5 @@
[TYPECHECK] [TYPECHECK]
ignored-modules = zmq ignored-modules = zmq
max-line-length = 150 max-line-length = 120
disable = missing-docstring, too-few-public-methods, invalid-name disable = missing-docstring, too-few-public-methods, invalid-name

View File

@ -80,4 +80,6 @@ The TFW message format:
Most of the components you need have docstrings included (hang on tight, this is work in progress) refer to them for usage info. Most of the components you need have docstrings included (hang on tight, this is work in progress) refer to them for usage info.
In the `docs` folder you can find our Sphinx-based API documentation, which you can build using the included `Makefile` (you need to have Sphinx installed, please reach out to us if you have trouble building the docs).
To get started you should take a look at the [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework) repository, which serves as an example project as well. To get started you should take a look at the [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework) repository, which serves as an example project as well.

View File

@ -1 +1 @@
egyptianmau bombay

20
docs/Makefile Normal file
View File

@ -0,0 +1,20 @@
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
SPHINXPROJ = baseimage-tutorial-framework
SOURCEDIR = source
BUILDDIR = build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

36
docs/make.bat Normal file
View File

@ -0,0 +1,36 @@
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=source
set BUILDDIR=build
set SPHINXPROJ=baseimage-tutorial-framework
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
:end
popd

View File

@ -0,0 +1,25 @@
Components
----------
.. automodule:: tfw.components
.. autoclass:: IdeEventHandler
:members:
.. autoclass:: TerminalEventHandler
:members:
.. autoclass:: ProcessManagingEventHandler
:members:
.. autoclass:: LogMonitoringEventHandler
:members:
.. autoclass:: TerminalCommands
:members:
.. autoclass:: HistoryMonitor
:members:
.. autoclass:: BashMonitor
:members:

166
docs/source/conf.py Normal file
View File

@ -0,0 +1,166 @@
# -*- coding: utf-8 -*-
#
# Configuration file for the Sphinx documentation builder.
#
# This file does only contain a selection of the most common options. For a
# full list see the documentation:
# http://www.sphinx-doc.org/en/master/config
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath('../../lib'))
# -- Project information -----------------------------------------------------
project = 'baseimage-tutorial-framework'
copyright = '2018, Avatao Innovative Learning Kft'
author = 'Kristóf Tóth'
# The short X.Y version
version = ''
# The full version, including alpha/beta/rc tags
release = 'bombay'
# -- General configuration ---------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path .
exclude_patterns = []
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# The default sidebars (for documents that don't match any pattern) are
# defined by theme itself. Builtin themes are using these templates by
# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
# 'searchbox.html']``.
#
# html_sidebars = {}
# -- Options for HTMLHelp output ---------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'baseimage-tutorial-frameworkdoc'
# -- Options for LaTeX output ------------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'baseimage-tutorial-framework.tex', 'baseimage-tutorial-framework Documentation',
'Kristóf Tóth', 'manual'),
]
# -- Options for manual page output ------------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'baseimage-tutorial-framework', 'baseimage-tutorial-framework Documentation',
[author], 1)
]
# -- Options for Texinfo output ----------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'baseimage-tutorial-framework', 'baseimage-tutorial-framework Documentation',
author, 'baseimage-tutorial-framework', 'One line description of project.',
'Miscellaneous'),
]
# -- Extension configuration -------------------------------------------------
def skip(app, what, name, obj, skip, options):
if name == "__init__":
return False
return skip
def setup(app):
app.connect("autodoc-skip-member", skip)

View File

@ -0,0 +1,12 @@
Event handler base classes
--------------------------
Subclass these to create your cusom event handlers.
.. automodule:: tfw
.. autoclass:: EventHandlerBase
:members:
.. autoclass:: TriggeredEventHandler
:members:

View File

@ -0,0 +1,12 @@
FSM base classes
----------------
Subclass these to create an FSM that fits your tutorial/challenge.
.. automodule:: tfw
.. autoclass:: FSMBase
:members:
.. autoclass:: LinearFSM
:members:

View File

@ -0,0 +1,7 @@
TFWServer
---------
.. automodule:: tfw.networking
.. autoclass:: TFWServer
:members:

45
docs/source/index.rst Normal file
View File

@ -0,0 +1,45 @@
.. baseimage-tutorial-framework documentation master file, created by
sphinx-quickstart on Fri Jun 1 14:29:07 2018.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to baseimage-tutorial-framework's documentation!
========================================================
Foundations
-----------
This part covers the soil which the framework is based on and stuff you will need to develop your own challenges.
.. toctree::
:glob:
foundations/*
Networking
----------
You can use these to send messages to the frontend or the event handlers through TFWServer.
.. toctree::
:glob:
networking/*
Components
----------
These are pre-written components for you to use, such as our IDE, terminal or console.
.. toctree::
:glob:
components/*
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

View File

@ -0,0 +1,17 @@
Networking
----------
.. automodule:: tfw.networking
.. autoclass:: TFWServerConnector
:members:
.. automodule:: tfw.networking.event_handlers
.. autoclass:: ServerUplinkConnector
:members:
.. automodule:: tfw.networking
.. autoclass:: MessageSender
:members:

View File

@ -4,9 +4,22 @@
from collections import namedtuple from collections import namedtuple
from os import environ from os import environ
from tfw.decorators import lazy_property
def prefixed_envvars_to_namedtuple(prefix: str, tuple_name: str):
envvars = {envvar.replace(prefix, '', 1): environ.get(envvar) class LazyEnvironment:
def __init__(self, prefix, tuple_name):
self._prefix = prefix
self._tuple_name = tuple_name
@lazy_property
def environment(self):
return self.prefixed_envvars_to_namedtuple()
def prefixed_envvars_to_namedtuple(self):
envvars = {
envvar.replace(self._prefix, '', 1): environ.get(envvar)
for envvar in environ.keys() for envvar in environ.keys()
if envvar.startswith(prefix)} if envvar.startswith(self._prefix)
return namedtuple(tuple_name, envvars)(**envvars) }
return namedtuple(self._tuple_name, envvars)(**envvars)

View File

@ -1,6 +1,6 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft. # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details. # All Rights Reserved. See LICENSE file for details.
from envvars import prefixed_envvars_to_namedtuple from envvars import LazyEnvironment
TAOENV = prefixed_envvars_to_namedtuple('AVATAO_', 'taoenvtuple') TAOENV = LazyEnvironment('AVATAO_', 'taoenvtuple').environment

View File

@ -16,7 +16,6 @@ LOG = logging.getLogger(__name__)
class DirectoryMonitor(ObserverMixin): class DirectoryMonitor(ObserverMixin):
def __init__(self, directories): def __init__(self, directories):
ObserverMixin.__init__(self)
self.eventhandler = IdeReloadWatchdogEventHandler() self.eventhandler = IdeReloadWatchdogEventHandler()
for directory in directories: for directory in directories:
self.observer.schedule(self.eventhandler, directory, recursive=True) self.observer.schedule(self.eventhandler, directory, recursive=True)
@ -65,8 +64,10 @@ class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler):
self.ignore = self.ignore - 1 self.ignore = self.ignore - 1
return return
LOG.debug(event) LOG.debug(event)
self.uplink.send({'key': 'ide', self.uplink.send({
'data': {'command': 'reload'}}) 'key': 'ide',
'data': {'command': 'reload'}
})
def with_monitor_paused(fun): def with_monitor_paused(fun):

View File

@ -17,10 +17,12 @@ class DirectoryMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin):
self._directory = directory self._directory = directory
MonitorManagerMixin.__init__(self, DirectoryMonitor, self._directory) MonitorManagerMixin.__init__(self, DirectoryMonitor, self._directory)
self.commands = {'pause': self.pause, self.commands = {
'pause': self.pause,
'resume': self.resume, 'resume': self.resume,
'ignore': self.ignore, 'ignore': self.ignore,
'selectdir': self.selectdir} 'selectdir': self.selectdir
}
@property @property
def directory(self): def directory(self):

View File

@ -36,15 +36,17 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC):
See examples below. See examples below.
""" """
def __init__(self, histfile): def __init__(self, histfile):
CallbackMixin.__init__(self)
ObserverMixin.__init__(self)
self.histfile = histfile self.histfile = histfile
self._history = [] self._history = []
self._last_length = len(self._history) self._last_length = len(self._history)
self.observer.schedule(CallbackEventHandler([self.histfile], self.observer.schedule(
CallbackEventHandler(
[self.histfile],
self._fetch_history, self._fetch_history,
self._invoke_callbacks), self._invoke_callbacks
dirname(self.histfile)) ),
dirname(self.histfile)
)
@property @property
def history(self): def history(self):
@ -55,7 +57,10 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC):
with open(self.histfile, 'r') as ifile: with open(self.histfile, 'r') as ifile:
pattern = compileregex(self.command_pattern) pattern = compileregex(self.command_pattern)
data = ifile.read() data = ifile.read()
self._history = [self.sanitize_command(command) for command in findall(pattern, data)] self._history = [
self.sanitize_command(command)
for command in findall(pattern, data)
]
@property @property
@abstractmethod @abstractmethod

View File

@ -41,8 +41,8 @@ class FileManager: # pylint: disable=too-many-instance-attributes
def workdir(self, directory): def workdir(self, directory):
if not exists(directory) or not isdir(directory): if not exists(directory) or not isdir(directory):
raise EnvironmentError(f'"{directory}" is not a directory!') raise EnvironmentError(f'"{directory}" is not a directory!')
if not self._is_in_whitelisted_dir(directory): if not self._is_in_allowed_dir(directory):
raise EnvironmentError(f'Directory "{directory}" is not in whitelist!') raise EnvironmentError(f'Directory "{directory}" is not allowed!')
self._workdir = directory self._workdir = directory
@property @property
@ -65,8 +65,13 @@ class FileManager: # pylint: disable=too-many-instance-attributes
@property @property
def files(self): def files(self):
return [self._relpath(file) for file in glob(join(self._workdir, '**/*'), recursive=True) return [
if isfile(file) and self._is_in_whitelisted_dir(file) and not self._is_blacklisted(file)] self._relpath(file)
for file in glob(join(self._workdir, '**/*'), recursive=True)
if isfile(file)
and self._is_in_allowed_dir(file)
and not self._is_blacklisted(file)
]
@property @property
def file_contents(self): def file_contents(self):
@ -78,11 +83,17 @@ class FileManager: # pylint: disable=too-many-instance-attributes
with open(self._filepath(self.filename), 'w', errors='surrogateescape') as ofile: with open(self._filepath(self.filename), 'w', errors='surrogateescape') as ofile:
ofile.write(value) ofile.write(value)
def _is_in_whitelisted_dir(self, path): def _is_in_allowed_dir(self, path):
return any(realpath(path).startswith(allowed_dir) for allowed_dir in self.allowed_directories) return any(
realpath(path).startswith(allowed_dir)
for allowed_dir in self.allowed_directories
)
def _is_blacklisted(self, file): def _is_blacklisted(self, file):
return any(fnmatchcase(file, blacklisted) for blacklisted in self.exclude) return any(
fnmatchcase(file, blacklisted)
for blacklisted in self.exclude
)
def _filepath(self, filename): def _filepath(self, filename):
return join(self._workdir, filename) return join(self._workdir, filename)
@ -101,45 +112,51 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
When any file in the selected directory changes they are automatically refreshed When any file in the selected directory changes they are automatically refreshed
on the frontend (this is done by listening to inotify events). on the frontend (this is done by listening to inotify events).
This EventHandler accepts messages that have a data["command"] key specifying This EventHandler accepts messages that have a data['command'] key specifying
a command to be executed. a command to be executed.
The API of each command is documented in their respective handlers.
The API of each command is documented in their respective handler.
""" """
def __init__(self, key, directory, allowed_directories, selected_file=None, exclude=None, def __init__(self, key, directory, allowed_directories, selected_file=None, exclude=None):
additional_watched_directories=None):
""" """
:param key: the key this instance should listen to :param key: the key this instance should listen to
:param directory: working directory which the EventHandler should serve files from :param directory: working directory which the EventHandler should serve files from
:param allowed_directories: list of directories that can be switched to using the selectdir command :param allowed_directories: list of directories that can be switched to using selectdir
:param selected_file: file that is selected by default :param selected_file: file that is selected by default
:param exclude: list of filenames that should not appear between files (for *.o, *.pyc, etc.) :param exclude: list of filenames that should not appear between files (for .o, .pyc, etc.)
:param additional_watched_directories: refresh the selected file when files change in these directories
(the working directory is watched by default, this is useful for
symlinks and such)
""" """
super().__init__(key) super().__init__(key)
try: try:
self.filemanager = FileManager(allowed_directories=allowed_directories, working_directory=directory, self.filemanager = FileManager(
selected_file=selected_file, exclude=exclude) allowed_directories=allowed_directories,
working_directory=directory,
selected_file=selected_file,
exclude=exclude
)
except IndexError: except IndexError:
raise EnvironmentError(f'No file(s) in IdeEventHandler working_directory "{directory}"!') raise EnvironmentError(
f'No file(s) in IdeEventHandler working_directory "{directory}"!'
)
self.watched_directories = [self.filemanager.workdir] MonitorManagerMixin.__init__(
if additional_watched_directories: self,
self.watched_directories.extend(additional_watched_directories) DirectoryMonitor,
MonitorManagerMixin.__init__(self, DirectoryMonitor, self.watched_directories) self.filemanager.allowed_directories
)
self.commands = {'read': self.read, self.commands = {
'read': self.read,
'write': self.write, 'write': self.write,
'select': self.select, 'select': self.select,
'selectdir': self.select_dir, 'selectdir': self.select_dir,
'exclude': self.exclude} 'exclude': self.exclude
}
def read(self, data): def read(self, data):
""" """
Read the currently selected file. Read the currently selected file.
:return: message with the contents of the file in data['content'] :return dict: message with the contents of the file in data['content']
""" """
try: try:
data['content'] = self.filemanager.file_contents data['content'] = self.filemanager.file_contents
@ -155,8 +172,9 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
""" """
Overwrites a file with the desired string. Overwrites a file with the desired string.
:param data: TFW message data containing keys: :param data: TFW message data containing key 'content'
|-string: containing the desired file contents (new file content)
""" """
self.monitor.ignore = self.monitor.ignore + 1 self.monitor.ignore = self.monitor.ignore + 1
try: try:
@ -170,8 +188,8 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
""" """
Selects a file from the current directory. Selects a file from the current directory.
:param data: TFW message data containing keys: :param data: TFW message data containing 'filename'
|-filename: name of file to select relative to the current directory (name of file to select relative to the current directory)
""" """
try: try:
self.filemanager.filename = data['filename'] self.filemanager.filename = data['filename']
@ -183,10 +201,10 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
""" """
Select a new working directory to display files from. Select a new working directory to display files from.
:param data: TFW message data containing keys: :param data: TFW message data containing 'directory'
|-directory: absolute path of diretory to select. (absolute path of diretory to select.
must be a path whitelisted in must be a path whitelisted in
self.allowed_directories self.allowed_directories)
""" """
try: try:
self.filemanager.workdir = data['directory'] self.filemanager.workdir = data['directory']
@ -204,8 +222,9 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
""" """
Overwrite list of excluded files Overwrite list of excluded files
:param data: TFW message data containing keys: :param data: TFW message data containing 'exclude'
|-exclude: list of filename patterns to be excluded, e.g.: ["*.pyc", "*.o"] (list of unix-style filename patterns to be excluded,
e.g.: ["\*.pyc", "\*.o")
""" """
try: try:
self.filemanager.exclude = list(data['exclude']) self.filemanager.exclude = list(data['exclude'])

View File

@ -14,8 +14,10 @@ from tfw.mixins import ObserverMixin, SupervisorLogMixin
class LogMonitor(ObserverMixin): class LogMonitor(ObserverMixin):
def __init__(self, process_name, log_tail=0): def __init__(self, process_name, log_tail=0):
self.prevent_log_recursion() self.prevent_log_recursion()
ObserverMixin.__init__(self) event_handler = SendLogWatchdogEventHandler(
event_handler = SendLogWatchdogEventHandler(process_name, log_tail=log_tail) process_name,
log_tail=log_tail
)
self.observer.schedule( self.observer.schedule(
event_handler, event_handler,
event_handler.path event_handler.path
@ -29,10 +31,12 @@ class LogMonitor(ObserverMixin):
class SendLogWatchdogEventHandler(PatternMatchingWatchdogEventHandler, SupervisorLogMixin): class SendLogWatchdogEventHandler(PatternMatchingWatchdogEventHandler, SupervisorLogMixin):
def __init__(self, process_name, log_tail=0): def __init__(self, process_name, log_tail=0):
self.acquire_own_supervisor_instance() # This thread-localises the xmlrpc client
self.process_name = process_name self.process_name = process_name
self.procinfo = self.supervisor.getProcessInfo(self.process_name) self.procinfo = self.supervisor.getProcessInfo(self.process_name)
super().__init__([self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']]) super().__init__([
self.procinfo['stdout_logfile'],
self.procinfo['stderr_logfile']
])
self.uplink = ServerUplinkConnector() self.uplink = ServerUplinkConnector()
self.log_tail = log_tail self.log_tail = log_tail

View File

@ -1,7 +1,6 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft. # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details. # All Rights Reserved. See LICENSE file for details.
from tfw import EventHandlerBase from tfw import EventHandlerBase
from tfw.mixins import MonitorManagerMixin from tfw.mixins import MonitorManagerMixin
from tfw.config.logs import logging from tfw.config.logs import logging
@ -11,11 +10,25 @@ LOG = logging.getLogger(__name__)
class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin):
"""
Monitors the output of a supervisor process (stdout, stderr) and
sends the results to the frontend.
Accepts messages that have a data['command'] key specifying
a command to be executed.
The API of each command is documented in their respective handler.
"""
def __init__(self, key, process_name, log_tail=0): def __init__(self, key, process_name, log_tail=0):
super().__init__(key) super().__init__(key)
self.process_name = process_name self.process_name = process_name
self.log_tail = log_tail self.log_tail = log_tail
MonitorManagerMixin.__init__(self, LogMonitor, self.process_name, self.log_tail) MonitorManagerMixin.__init__(
self,
LogMonitor,
self.process_name,
self.log_tail
)
self.command_handlers = { self.command_handlers = {
'process_name': self.handle_process_name, 'process_name': self.handle_process_name,
@ -31,7 +44,24 @@ class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin):
LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
def handle_process_name(self, data): def handle_process_name(self, data):
self.set_monitor_args(data['process_name'], self.log_tail) """
Changes the monitored process.
:param data: TFW message data containing 'value'
(name of the process to monitor)
"""
self.set_monitor_args(data['value'], self.log_tail)
def handle_log_tail(self, data): def handle_log_tail(self, data):
self.set_monitor_args(self.process_name, data['log_tail']) """
Sets tail length of the log the monitor will send
to the frontend (the monitor will send back the last
'value' characters of the log).
:param data: TFW message data containing 'value'
(new tail length)
"""
self.set_monitor_args(self.process_name, data['value'])
def cleanup(self):
self.monitor.stop()

View File

@ -13,9 +13,11 @@ LOG = logging.getLogger(__name__)
class ProcessManager(SupervisorMixin, SupervisorLogMixin): class ProcessManager(SupervisorMixin, SupervisorLogMixin):
def __init__(self): def __init__(self):
self.commands = {'start': self.start_process, self.commands = {
'start': self.start_process,
'stop': self.stop_process, 'stop': self.stop_process,
'restart': self.restart_process} 'restart': self.restart_process
}
def __call__(self, command, process_name): def __call__(self, command, process_name):
return self.commands[command](process_name) return self.commands[command](process_name)
@ -25,9 +27,9 @@ class ProcessManagingEventHandler(EventHandlerBase):
""" """
Event handler that can manage processes managed by supervisor. Event handler that can manage processes managed by supervisor.
This EventHandler accepts messages that have a data["command"] key specifying This EventHandler accepts messages that have a data['command'] key specifying
a command to be executed. a command to be executed.
Every message must contain a data["process_name"] field with the name of the Every message must contain a data['process_name'] field with the name of the
process to manage. This is the name specified in supervisor config files like so: process to manage. This is the name specified in supervisor config files like so:
[program:someprogram] [program:someprogram]
@ -50,8 +52,14 @@ class ProcessManagingEventHandler(EventHandlerBase):
except SupervisorFault as fault: except SupervisorFault as fault:
message['data']['error'] = fault.faultString message['data']['error'] = fault.faultString
finally: finally:
message['data']['stdout'] = self.processmanager.read_stdout(data['process_name'], self.log_tail) message['data']['stdout'] = self.processmanager.read_stdout(
message['data']['stderr'] = self.processmanager.read_stderr(data['process_name'], self.log_tail) data['process_name'],
self.log_tail
)
message['data']['stderr'] = self.processmanager.read_stderr(
data['process_name'],
self.log_tail
)
return message return message
except KeyError: except KeyError:
LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)

View File

@ -14,15 +14,15 @@ LOG = logging.getLogger(__name__)
class TerminadoMiniServer: class TerminadoMiniServer:
def __init__(self, url, port, workdir, shellcmd): def __init__(self, url, port, workdir, shellcmd):
self.port = port self.port = port
self._term_manager = SingleTermManager(shell_command=shellcmd, self._term_manager = SingleTermManager(
term_settings={'cwd': workdir}) shell_command=shellcmd,
self.application = Application( term_settings={'cwd': workdir}
[( )
self.application = Application([(
url, url,
TerminadoMiniServer.ResetterTermSocket, TerminadoMiniServer.ResetterTermSocket,
{'term_manager': self._term_manager} {'term_manager': self._term_manager}
)] )])
)
@property @property
def term_manager(self): def term_manager(self):
@ -46,5 +46,10 @@ class TerminadoMiniServer:
if __name__ == '__main__': if __name__ == '__main__':
LOG.info('Terminado Mini Server listening on %s', TFWENV.TERMINADO_PORT) LOG.info('Terminado Mini Server listening on %s', TFWENV.TERMINADO_PORT)
TerminadoMiniServer('/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, ['bash']).listen() TerminadoMiniServer(
'/terminal',
TFWENV.TERMINADO_PORT,
TFWENV.TERMINADO_WD,
['bash']
).listen()
IOLoop.instance().start() IOLoop.instance().start()

View File

@ -19,13 +19,13 @@ class TerminalCommands(ABC):
To receive events you need to subscribe TerminalCommand.callback to a HistoryMonitor To receive events you need to subscribe TerminalCommand.callback to a HistoryMonitor
instance. instance.
Inherit from this class and define methods which start with "command_". When the user Inherit from this class and define methods which start with "command\_". When the user
executes the command specified after the underscore, your method will be invoked. All executes the command specified after the underscore, your method will be invoked. All
such commands must expect the parameter *args which will contain the arguments of the such commands must expect the parameter \*args which will contain the arguments of the
command. command.
For example to define a method that runs when someone starts vim in the terminal For example to define a method that runs when someone starts vim in the terminal
you have to define a method like: "def command_vim(self, *args)" you have to define a method like: "def command_vim(self, \*args)"
You can also use this class to create new commands similarly. You can also use this class to create new commands similarly.
""" """
@ -36,8 +36,12 @@ class TerminalCommands(ABC):
self._setup_bashrc_aliases(bashrc) self._setup_bashrc_aliases(bashrc)
def _build_command_to_implementation_dict(self): def _build_command_to_implementation_dict(self):
return {self._parse_command_name(fun): getattr(self, fun) for fun in dir(self) return {
if callable(getattr(self, fun)) and self._is_command_implementation(fun)} self._parse_command_name(fun): getattr(self, fun)
for fun in dir(self)
if callable(getattr(self, fun))
and self._is_command_implementation(fun)
}
def _setup_bashrc_aliases(self, bashrc): def _setup_bashrc_aliases(self, bashrc):
with open(bashrc, 'a') as ofile: with open(bashrc, 'a') as ofile:

View File

@ -16,9 +16,9 @@ class TerminalEventHandler(EventHandlerBase):
sessions to connect to. You need to instanciate this in order for frontend sessions to connect to. You need to instanciate this in order for frontend
terminals to work. terminals to work.
This EventHandler accepts messages that have a data["command"] key specifying This EventHandler accepts messages that have a data['command'] key specifying
a command to be executed. a command to be executed.
The API of each command is documented in their respective handlers. The API of each command is documented in their respective handler.
""" """
def __init__(self, key, monitor): def __init__(self, key, monitor):
""" """
@ -29,9 +29,19 @@ class TerminalEventHandler(EventHandlerBase):
self.working_directory = TFWENV.TERMINADO_DIR self.working_directory = TFWENV.TERMINADO_DIR
self._historymonitor = monitor self._historymonitor = monitor
bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash'] bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash']
self.terminado_server = TerminadoMiniServer('/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, bash_as_user_cmd)
self.commands = {'write': self.write, self.terminado_server = TerminadoMiniServer(
'read': self.read} '/terminal',
TFWENV.TERMINADO_PORT,
TFWENV.TERMINADO_WD,
bash_as_user_cmd
)
self.commands = {
'write': self.write,
'read': self.read
}
if self._historymonitor: if self._historymonitor:
self._historymonitor.watch() self._historymonitor.watch()
self.terminado_server.listen() self.terminado_server.listen()
@ -54,18 +64,18 @@ class TerminalEventHandler(EventHandlerBase):
Writes a string to the terminal session (on the pty level). Writes a string to the terminal session (on the pty level).
Useful for pre-typing and executing commands for the user. Useful for pre-typing and executing commands for the user.
:param data: TFW message data containing keys: :param data: TFW message data containing 'value'
|-shellcmd: command to be written to the pty (command to be written to the pty)
""" """
self.terminado_server.pty.write(data['shellcmd']) self.terminado_server.pty.write(data['value'])
def read(self, data): def read(self, data):
""" """
Reads the history of commands executed. Reads the history of commands executed.
:param data: TFW message data containing keys: :param data: TFW message data containing 'count'
|-count: the number of history elements to return (the number of history elements to return)
:return: message with list of commands in data['history'] :return dict: message with list of commands in data['history']
""" """
data['count'] = int(data.get('count', 1)) data['count'] = int(data.get('count', 1))
if self.historymonitor: if self.historymonitor:

View File

@ -1,6 +1,6 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft. # Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details. # All Rights Reserved. See LICENSE file for details.
from envvars import prefixed_envvars_to_namedtuple from envvars import LazyEnvironment
TFWENV = prefixed_envvars_to_namedtuple('TFW_', 'tfwenvtuple') TFWENV = LazyEnvironment('TFW_', 'tfwenvtuple').environment

View File

@ -2,3 +2,4 @@
# All Rights Reserved. See LICENSE file for details. # All Rights Reserved. See LICENSE file for details.
from .rate_limiter import RateLimiter from .rate_limiter import RateLimiter
from .lazy_property import lazy_property

View File

@ -0,0 +1,19 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
class lazy_property:
"""
Decorator that replaces a function with the value
it calculates on the first call.
"""
def __init__(self, func):
self.func = func
self.__doc__ = func.__doc__
def __get__(self, instance, owner):
if instance is None:
return self # avoids potential __new__ TypeError
value = self.func(instance)
setattr(instance, self.func.__name__, value)
return value

View File

@ -19,15 +19,16 @@ class FSMBase(CallbackMixin):
states, transitions = [], [] states, transitions = [], []
def __init__(self, initial: str = None, accepted_states: List[str] = None): def __init__(self, initial: str = None, accepted_states: List[str] = None):
CallbackMixin.__init__(self)
self.accepted_states = accepted_states or [self.states[-1]] self.accepted_states = accepted_states or [self.states[-1]]
self.machine = Machine(model=self, self.machine = Machine(
model=self,
states=self.states, states=self.states,
transitions=self.transitions, transitions=self.transitions,
initial=initial or self.states[0], initial=initial or self.states[0],
send_event=True, send_event=True,
ignore_invalid_triggers=True, ignore_invalid_triggers=True,
after_state_change='execute_callbacks') after_state_change='execute_callbacks'
)
def execute_callbacks(self, event_data): def execute_callbacks(self, event_data):
self._execute_callbacks(event_data.kwargs) self._execute_callbacks(event_data.kwargs)

View File

@ -11,12 +11,19 @@ class LinearFSM(FSMBase):
a number of steps specified in the constructor. It automatically sets up 2 a number of steps specified in the constructor. It automatically sets up 2
actions (triggers) between states as such: actions (triggers) between states as such:
(0) -- step_1 --> (1) -- step_2 --> (2) -- step_3 --> (3) ... and so on (0) -- step_1 --> (1) -- step_2 --> (2) -- step_3 --> (3) ... and so on
\-step_next-/ \-step_next-/ \-step_next-/
""" """
def __init__(self, number_of_steps): def __init__(self, number_of_steps):
self.states = list(map(str, range(number_of_steps))) self.states = list(map(str, range(number_of_steps)))
self.transitions = [] self.transitions = []
for index in self.states[:-1]: for index in self.states[:-1]:
self.transitions.append({'trigger': f'step_{int(index)+1}', 'source': index, 'dest': str(int(index)+1)}) self.transitions.append({
self.transitions.append({'trigger': 'step_next', 'source': index, 'dest': str(int(index)+1)}) 'trigger': f'step_{int(index)+1}',
'source': index,
'dest': str(int(index)+1)
})
self.transitions.append({
'trigger': 'step_next',
'source': index,
'dest': str(int(index)+1)
})
super(LinearFSM, self).__init__() super(LinearFSM, self).__init__()

View File

@ -3,17 +3,20 @@
from functools import partial from functools import partial
from tfw.decorators import lazy_property
class CallbackMixin: class CallbackMixin:
def __init__(self): @lazy_property
self._callbacks = [] def _callbacks(self):
return []
def subscribe_callback(self, callback, *args, **kwargs): def subscribe_callback(self, callback, *args, **kwargs):
""" """
Subscribe a callable to invoke once an event is triggered. Subscribe a callable to invoke once an event is triggered.
:param callback: callable to be executed on events :param callback: callable to be executed on events
:param *args: arguments passed to callable :param args: arguments passed to callable
:param **kwargs: kwargs passed to callable :param kwargs: kwargs passed to callable
""" """
fun = partial(callback, *args, **kwargs) fun = partial(callback, *args, **kwargs)
self._callbacks.append(fun) self._callbacks.append(fun)

View File

@ -3,10 +3,13 @@
from watchdog.observers import Observer from watchdog.observers import Observer
from tfw.decorators import lazy_property
class ObserverMixin: class ObserverMixin:
def __init__(self): @lazy_property
self.observer = Observer() def observer(self):
return Observer()
def watch(self): def watch(self):
self.observer.start() self.observer.start()

View File

@ -6,21 +6,14 @@ from xmlrpc.client import Fault as SupervisorFault
from contextlib import suppress from contextlib import suppress
from os import remove from os import remove
from tfw.decorators import lazy_property
from tfw.config import TFWENV from tfw.config import TFWENV
def get_supervisor_instance():
return xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor
class SupervisorBaseMixin: class SupervisorBaseMixin:
supervisor = get_supervisor_instance() @lazy_property
def supervisor(self):
def acquire_own_supervisor_instance(self): return xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor
"""
Give this instance non-static, local xmlrpc client
"""
self.supervisor = get_supervisor_instance()
class SupervisorMixin(SupervisorBaseMixin): class SupervisorMixin(SupervisorBaseMixin):

View File

@ -26,5 +26,7 @@ class MessageSender:
'timestamp': datetime.now().isoformat(), 'timestamp': datetime.now().isoformat(),
'message': message 'message': message
} }
self.server_connector.send({'key': self.key, self.server_connector.send({
'data': data}) 'key': self.key,
'data': data
})

View File

@ -43,11 +43,17 @@ def deserialize_tfw_msg(*args):
def _serialize_all(*args): def _serialize_all(*args):
return tuple(_serialize_single(arg) for arg in args) return tuple(
_serialize_single(arg)
for arg in args
)
def _deserialize_all(*args): def _deserialize_all(*args):
return tuple(_deserialize_single(arg) for arg in args) return tuple(
_deserialize_single(arg)
for arg in args
)
def _serialize_single(data): def _serialize_single(data):

View File

@ -30,13 +30,16 @@ class TFWServer:
self._fsm.subscribe_callback(self._fsm_updater.update) self._fsm.subscribe_callback(self._fsm_updater.update)
self._event_handler_connector = EventHandlerConnector() self._event_handler_connector = EventHandlerConnector()
self.application = Application( self.application = Application([(
[(r'/ws', ZMQWebSocketProxy, {'make_eventhandler_message': self.make_eventhandler_message, r'/ws', ZMQWebSocketProxy,{
'make_eventhandler_message': self.make_eventhandler_message,
'proxy_filter': self.proxy_filter, 'proxy_filter': self.proxy_filter,
'handle_trigger': self.handle_trigger, 'handle_trigger': self.handle_trigger,
'event_handler_connector': self._event_handler_connector})] 'event_handler_connector': self._event_handler_connector
})]
) )
#self.controller_responder = ControllerResponder(self.fsm) TODO: add this once controller stuff is resolved # self.controller_responder = ControllerResponder(self.fsm)
# TODO: add this once controller stuff is resolved
@property @property
def fsm(self): def fsm(self):
@ -97,8 +100,11 @@ class FSMManager:
self.trigger_predicates[trigger].extend(predicates) self.trigger_predicates[trigger].extend(predicates)
def unsubscribe_predicate(self, trigger, *predicates): def unsubscribe_predicate(self, trigger, *predicates):
self.trigger_predicates[trigger] = [predicate for predicate in self.trigger_predicates[trigger] self.trigger_predicates[trigger] = [
not in predicates] predicate
for predicate in self.trigger_predicates[trigger]
not in predicates
]
class FSMUpdater: class FSMUpdater:
@ -111,10 +117,18 @@ class FSMUpdater:
self.uplink.send(self.generate_fsm_update()) self.uplink.send(self.generate_fsm_update())
def generate_fsm_update(self): def generate_fsm_update(self):
return {'key': 'FSMUpdate', return {
'data': self.get_fsm_state_and_transitions()} 'key': 'FSMUpdate',
'data': self.get_fsm_state_and_transitions()
}
def get_fsm_state_and_transitions(self): def get_fsm_state_and_transitions(self):
state = self.fsm.state state = self.fsm.state
valid_transitions = [{'trigger': trigger} for trigger in self.fsm.machine.get_triggers(self.fsm.state)] valid_transitions = [
return {'current_state': state, 'valid_transitions': valid_transitions} {'trigger': trigger}
for trigger in self.fsm.machine.get_triggers(self.fsm.state)
]
return {
'current_state': state,
'valid_transitions': valid_transitions
}