diff --git a/.drone.yml b/.drone.yml index 9e21f9e..0b24068 100644 --- a/.drone.yml +++ b/.drone.yml @@ -10,4 +10,4 @@ pipeline: - docker push eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG} when: event: 'tag' - branch: refs/tags/egyptianmau-20* + branch: refs/tags/bombay-20* diff --git a/.pylintrc b/.pylintrc index 7fb46c0..f36e237 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,5 +1,5 @@ [TYPECHECK] ignored-modules = zmq -max-line-length = 150 +max-line-length = 120 disable = missing-docstring, too-few-public-methods, invalid-name diff --git a/README.md b/README.md index e3b7f54..01b6933 100644 --- a/README.md +++ b/README.md @@ -80,4 +80,6 @@ The TFW message format: Most of the components you need have docstrings included (hang on tight, this is work in progress) – refer to them for usage info. +In the `docs` folder you can find our Sphinx-based API documentation, which you can build using the included `Makefile` (you need to have Sphinx installed, please reach out to us if you have trouble building the docs). + To get started you should take a look at the [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework) repository, which serves as an example project as well. diff --git a/VERSION b/VERSION index 23d699d..4958238 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -egyptianmau +bombay diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..1a264d7 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = baseimage-tutorial-framework +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..0a0210b --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build +set SPHINXPROJ=baseimage-tutorial-framework + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/source/components/components.rst b/docs/source/components/components.rst new file mode 100644 index 0000000..87dee5b --- /dev/null +++ b/docs/source/components/components.rst @@ -0,0 +1,25 @@ +Components +---------- + +.. automodule:: tfw.components + +.. autoclass:: IdeEventHandler + :members: + +.. autoclass:: TerminalEventHandler + :members: + +.. autoclass:: ProcessManagingEventHandler + :members: + +.. autoclass:: LogMonitoringEventHandler + :members: + +.. autoclass:: TerminalCommands + :members: + +.. autoclass:: HistoryMonitor + :members: + +.. autoclass:: BashMonitor + :members: diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..5c596d0 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../../lib')) + + +# -- Project information ----------------------------------------------------- + +project = 'baseimage-tutorial-framework' +copyright = '2018, Avatao Innovative Learning Kft' +author = 'Kristóf Tóth' + +# The short X.Y version +version = '' +# The full version, including alpha/beta/rc tags +release = 'bombay' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'baseimage-tutorial-frameworkdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'baseimage-tutorial-framework.tex', 'baseimage-tutorial-framework Documentation', + 'Kristóf Tóth', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'baseimage-tutorial-framework', 'baseimage-tutorial-framework Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'baseimage-tutorial-framework', 'baseimage-tutorial-framework Documentation', + author, 'baseimage-tutorial-framework', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Extension configuration ------------------------------------------------- +def skip(app, what, name, obj, skip, options): + if name == "__init__": + return False + return skip + +def setup(app): + app.connect("autodoc-skip-member", skip) diff --git a/docs/source/foundations/eventhandlers.rst b/docs/source/foundations/eventhandlers.rst new file mode 100644 index 0000000..9b8d91c --- /dev/null +++ b/docs/source/foundations/eventhandlers.rst @@ -0,0 +1,12 @@ +Event handler base classes +-------------------------- + +Subclass these to create your cusom event handlers. + +.. automodule:: tfw + +.. autoclass:: EventHandlerBase + :members: + +.. autoclass:: TriggeredEventHandler + :members: diff --git a/docs/source/foundations/fsms.rst b/docs/source/foundations/fsms.rst new file mode 100644 index 0000000..190ee8b --- /dev/null +++ b/docs/source/foundations/fsms.rst @@ -0,0 +1,12 @@ +FSM base classes +---------------- + +Subclass these to create an FSM that fits your tutorial/challenge. + +.. automodule:: tfw + +.. autoclass:: FSMBase + :members: + +.. autoclass:: LinearFSM + :members: diff --git a/docs/source/foundations/tfwserver.rst b/docs/source/foundations/tfwserver.rst new file mode 100644 index 0000000..3dddaa4 --- /dev/null +++ b/docs/source/foundations/tfwserver.rst @@ -0,0 +1,7 @@ +TFWServer +--------- + +.. automodule:: tfw.networking + +.. autoclass:: TFWServer + :members: diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..3d674ad --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,45 @@ +.. baseimage-tutorial-framework documentation master file, created by + sphinx-quickstart on Fri Jun 1 14:29:07 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to baseimage-tutorial-framework's documentation! +======================================================== + +Foundations +----------- + +This part covers the soil which the framework is based on and stuff you will need to develop your own challenges. + +.. toctree:: + :glob: + + foundations/* + +Networking +---------- + +You can use these to send messages to the frontend or the event handlers through TFWServer. + +.. toctree:: + :glob: + + networking/* + +Components +---------- + +These are pre-written components for you to use, such as our IDE, terminal or console. + +.. toctree:: + :glob: + + components/* + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/networking/networking.rst b/docs/source/networking/networking.rst new file mode 100644 index 0000000..317d266 --- /dev/null +++ b/docs/source/networking/networking.rst @@ -0,0 +1,17 @@ +Networking +---------- + +.. automodule:: tfw.networking + +.. autoclass:: TFWServerConnector + :members: + +.. automodule:: tfw.networking.event_handlers + +.. autoclass:: ServerUplinkConnector + :members: + +.. automodule:: tfw.networking + +.. autoclass:: MessageSender + :members: diff --git a/lib/envvars/__init__.py b/lib/envvars/__init__.py index 712a926..d1ed0d4 100644 --- a/lib/envvars/__init__.py +++ b/lib/envvars/__init__.py @@ -4,9 +4,22 @@ from collections import namedtuple from os import environ +from tfw.decorators import lazy_property -def prefixed_envvars_to_namedtuple(prefix: str, tuple_name: str): - envvars = {envvar.replace(prefix, '', 1): environ.get(envvar) - for envvar in environ.keys() - if envvar.startswith(prefix)} - return namedtuple(tuple_name, envvars)(**envvars) + +class LazyEnvironment: + def __init__(self, prefix, tuple_name): + self._prefix = prefix + self._tuple_name = tuple_name + + @lazy_property + def environment(self): + return self.prefixed_envvars_to_namedtuple() + + def prefixed_envvars_to_namedtuple(self): + envvars = { + envvar.replace(self._prefix, '', 1): environ.get(envvar) + for envvar in environ.keys() + if envvar.startswith(self._prefix) + } + return namedtuple(self._tuple_name, envvars)(**envvars) diff --git a/lib/tao/config/envvars.py b/lib/tao/config/envvars.py index da46222..260d057 100644 --- a/lib/tao/config/envvars.py +++ b/lib/tao/config/envvars.py @@ -1,6 +1,6 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from envvars import prefixed_envvars_to_namedtuple +from envvars import LazyEnvironment -TAOENV = prefixed_envvars_to_namedtuple('AVATAO_', 'taoenvtuple') +TAOENV = LazyEnvironment('AVATAO_', 'taoenvtuple').environment diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index ba423be..dc72e9e 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -16,7 +16,6 @@ LOG = logging.getLogger(__name__) class DirectoryMonitor(ObserverMixin): def __init__(self, directories): - ObserverMixin.__init__(self) self.eventhandler = IdeReloadWatchdogEventHandler() for directory in directories: self.observer.schedule(self.eventhandler, directory, recursive=True) @@ -65,8 +64,10 @@ class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler): self.ignore = self.ignore - 1 return LOG.debug(event) - self.uplink.send({'key': 'ide', - 'data': {'command': 'reload'}}) + self.uplink.send({ + 'key': 'ide', + 'data': {'command': 'reload'} + }) def with_monitor_paused(fun): diff --git a/lib/tfw/components/directory_monitoring_event_handler.py b/lib/tfw/components/directory_monitoring_event_handler.py index 03f58e1..b3beeb2 100644 --- a/lib/tfw/components/directory_monitoring_event_handler.py +++ b/lib/tfw/components/directory_monitoring_event_handler.py @@ -17,10 +17,12 @@ class DirectoryMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): self._directory = directory MonitorManagerMixin.__init__(self, DirectoryMonitor, self._directory) - self.commands = {'pause': self.pause, - 'resume': self.resume, - 'ignore': self.ignore, - 'selectdir': self.selectdir} + self.commands = { + 'pause': self.pause, + 'resume': self.resume, + 'ignore': self.ignore, + 'selectdir': self.selectdir + } @property def directory(self): diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index d5586a7..9d259ae 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -36,15 +36,17 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC): See examples below. """ def __init__(self, histfile): - CallbackMixin.__init__(self) - ObserverMixin.__init__(self) self.histfile = histfile self._history = [] self._last_length = len(self._history) - self.observer.schedule(CallbackEventHandler([self.histfile], - self._fetch_history, - self._invoke_callbacks), - dirname(self.histfile)) + self.observer.schedule( + CallbackEventHandler( + [self.histfile], + self._fetch_history, + self._invoke_callbacks + ), + dirname(self.histfile) + ) @property def history(self): @@ -55,7 +57,10 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC): with open(self.histfile, 'r') as ifile: pattern = compileregex(self.command_pattern) data = ifile.read() - self._history = [self.sanitize_command(command) for command in findall(pattern, data)] + self._history = [ + self.sanitize_command(command) + for command in findall(pattern, data) + ] @property @abstractmethod @@ -76,10 +81,10 @@ class BashMonitor(HistoryMonitor): HistoryMonitor for monitoring bash CLI sessions. This requires the following to be set in bash (note that this is done automatically by TFW): - PROMPT_COMMAND="history -a" - shopt -s cmdhist - shopt -s histappend - unset HISTCONTROL + PROMPT_COMMAND="history -a" + shopt -s cmdhist + shopt -s histappend + unset HISTCONTROL """ @property def command_pattern(self): diff --git a/lib/tfw/components/ide_event_handler.py b/lib/tfw/components/ide_event_handler.py index 7c9fda3..659ade0 100644 --- a/lib/tfw/components/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler.py @@ -41,8 +41,8 @@ class FileManager: # pylint: disable=too-many-instance-attributes def workdir(self, directory): if not exists(directory) or not isdir(directory): raise EnvironmentError(f'"{directory}" is not a directory!') - if not self._is_in_whitelisted_dir(directory): - raise EnvironmentError(f'Directory "{directory}" is not in whitelist!') + if not self._is_in_allowed_dir(directory): + raise EnvironmentError(f'Directory "{directory}" is not allowed!') self._workdir = directory @property @@ -65,8 +65,13 @@ class FileManager: # pylint: disable=too-many-instance-attributes @property def files(self): - return [self._relpath(file) for file in glob(join(self._workdir, '**/*'), recursive=True) - if isfile(file) and self._is_in_whitelisted_dir(file) and not self._is_blacklisted(file)] + return [ + self._relpath(file) + for file in glob(join(self._workdir, '**/*'), recursive=True) + if isfile(file) + and self._is_in_allowed_dir(file) + and not self._is_blacklisted(file) + ] @property def file_contents(self): @@ -78,11 +83,17 @@ class FileManager: # pylint: disable=too-many-instance-attributes with open(self._filepath(self.filename), 'w', errors='surrogateescape') as ofile: ofile.write(value) - def _is_in_whitelisted_dir(self, path): - return any(realpath(path).startswith(allowed_dir) for allowed_dir in self.allowed_directories) + def _is_in_allowed_dir(self, path): + return any( + realpath(path).startswith(allowed_dir) + for allowed_dir in self.allowed_directories + ) def _is_blacklisted(self, file): - return any(fnmatchcase(file, blacklisted) for blacklisted in self.exclude) + return any( + fnmatchcase(file, blacklisted) + for blacklisted in self.exclude + ) def _filepath(self, filename): return join(self._workdir, filename) @@ -101,45 +112,51 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): When any file in the selected directory changes they are automatically refreshed on the frontend (this is done by listening to inotify events). - This EventHandler accepts messages that have a data["command"] key specifying + This EventHandler accepts messages that have a data['command'] key specifying a command to be executed. - The API of each command is documented in their respective handlers. + + The API of each command is documented in their respective handler. """ - def __init__(self, key, directory, allowed_directories, selected_file=None, exclude=None, - additional_watched_directories=None): + def __init__(self, key, directory, allowed_directories, selected_file=None, exclude=None): """ :param key: the key this instance should listen to :param directory: working directory which the EventHandler should serve files from - :param allowed_directories: list of directories that can be switched to using the selectdir command + :param allowed_directories: list of directories that can be switched to using selectdir :param selected_file: file that is selected by default - :param exclude: list of filenames that should not appear between files (for *.o, *.pyc, etc.) - :param additional_watched_directories: refresh the selected file when files change in these directories - (the working directory is watched by default, this is useful for - symlinks and such) + :param exclude: list of filenames that should not appear between files (for .o, .pyc, etc.) """ super().__init__(key) try: - self.filemanager = FileManager(allowed_directories=allowed_directories, working_directory=directory, - selected_file=selected_file, exclude=exclude) + self.filemanager = FileManager( + allowed_directories=allowed_directories, + working_directory=directory, + selected_file=selected_file, + exclude=exclude + ) except IndexError: - raise EnvironmentError(f'No file(s) in IdeEventHandler working_directory "{directory}"!') + raise EnvironmentError( + f'No file(s) in IdeEventHandler working_directory "{directory}"!' + ) - self.watched_directories = [self.filemanager.workdir] - if additional_watched_directories: - self.watched_directories.extend(additional_watched_directories) - MonitorManagerMixin.__init__(self, DirectoryMonitor, self.watched_directories) + MonitorManagerMixin.__init__( + self, + DirectoryMonitor, + self.filemanager.allowed_directories + ) - self.commands = {'read': self.read, - 'write': self.write, - 'select': self.select, - 'selectdir': self.select_dir, - 'exclude': self.exclude} + self.commands = { + 'read': self.read, + 'write': self.write, + 'select': self.select, + 'selectdir': self.select_dir, + 'exclude': self.exclude + } def read(self, data): """ Read the currently selected file. - :return: message with the contents of the file in data['content'] + :return dict: message with the contents of the file in data['content'] """ try: data['content'] = self.filemanager.file_contents @@ -155,8 +172,9 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): """ Overwrites a file with the desired string. - :param data: TFW message data containing keys: - |-string: containing the desired file contents + :param data: TFW message data containing key 'content' + (new file content) + """ self.monitor.ignore = self.monitor.ignore + 1 try: @@ -170,8 +188,8 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): """ Selects a file from the current directory. - :param data: TFW message data containing keys: - |-filename: name of file to select relative to the current directory + :param data: TFW message data containing 'filename' + (name of file to select relative to the current directory) """ try: self.filemanager.filename = data['filename'] @@ -183,10 +201,10 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): """ Select a new working directory to display files from. - :param data: TFW message data containing keys: - |-directory: absolute path of diretory to select. - must be a path whitelisted in - self.allowed_directories + :param data: TFW message data containing 'directory' + (absolute path of diretory to select. + must be a path whitelisted in + self.allowed_directories) """ try: self.filemanager.workdir = data['directory'] @@ -204,8 +222,9 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): """ Overwrite list of excluded files - :param data: TFW message data containing keys: - |-exclude: list of filename patterns to be excluded, e.g.: ["*.pyc", "*.o"] + :param data: TFW message data containing 'exclude' + (list of unix-style filename patterns to be excluded, + e.g.: ["\*.pyc", "\*.o") """ try: self.filemanager.exclude = list(data['exclude']) diff --git a/lib/tfw/components/log_monitor.py b/lib/tfw/components/log_monitor.py index 287618f..2c6ade8 100644 --- a/lib/tfw/components/log_monitor.py +++ b/lib/tfw/components/log_monitor.py @@ -14,8 +14,10 @@ from tfw.mixins import ObserverMixin, SupervisorLogMixin class LogMonitor(ObserverMixin): def __init__(self, process_name, log_tail=0): self.prevent_log_recursion() - ObserverMixin.__init__(self) - event_handler = SendLogWatchdogEventHandler(process_name, log_tail=log_tail) + event_handler = SendLogWatchdogEventHandler( + process_name, + log_tail=log_tail + ) self.observer.schedule( event_handler, event_handler.path @@ -29,10 +31,12 @@ class LogMonitor(ObserverMixin): class SendLogWatchdogEventHandler(PatternMatchingWatchdogEventHandler, SupervisorLogMixin): def __init__(self, process_name, log_tail=0): - self.acquire_own_supervisor_instance() # This thread-localises the xmlrpc client self.process_name = process_name self.procinfo = self.supervisor.getProcessInfo(self.process_name) - super().__init__([self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']]) + super().__init__([ + self.procinfo['stdout_logfile'], + self.procinfo['stderr_logfile'] + ]) self.uplink = ServerUplinkConnector() self.log_tail = log_tail diff --git a/lib/tfw/components/log_monitoring_event_handler.py b/lib/tfw/components/log_monitoring_event_handler.py index e28a3ad..c5dd84c 100644 --- a/lib/tfw/components/log_monitoring_event_handler.py +++ b/lib/tfw/components/log_monitoring_event_handler.py @@ -1,7 +1,6 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. - from tfw import EventHandlerBase from tfw.mixins import MonitorManagerMixin from tfw.config.logs import logging @@ -11,11 +10,25 @@ LOG = logging.getLogger(__name__) class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): + """ + Monitors the output of a supervisor process (stdout, stderr) and + sends the results to the frontend. + + Accepts messages that have a data['command'] key specifying + a command to be executed. + + The API of each command is documented in their respective handler. + """ def __init__(self, key, process_name, log_tail=0): super().__init__(key) self.process_name = process_name self.log_tail = log_tail - MonitorManagerMixin.__init__(self, LogMonitor, self.process_name, self.log_tail) + MonitorManagerMixin.__init__( + self, + LogMonitor, + self.process_name, + self.log_tail + ) self.command_handlers = { 'process_name': self.handle_process_name, @@ -31,7 +44,24 @@ class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) def handle_process_name(self, data): - self.set_monitor_args(data['process_name'], self.log_tail) + """ + Changes the monitored process. + + :param data: TFW message data containing 'value' + (name of the process to monitor) + """ + self.set_monitor_args(data['value'], self.log_tail) def handle_log_tail(self, data): - self.set_monitor_args(self.process_name, data['log_tail']) + """ + Sets tail length of the log the monitor will send + to the frontend (the monitor will send back the last + 'value' characters of the log). + + :param data: TFW message data containing 'value' + (new tail length) + """ + self.set_monitor_args(self.process_name, data['value']) + + def cleanup(self): + self.monitor.stop() diff --git a/lib/tfw/components/process_managing_event_handler.py b/lib/tfw/components/process_managing_event_handler.py index a512c1e..3ccf6b1 100644 --- a/lib/tfw/components/process_managing_event_handler.py +++ b/lib/tfw/components/process_managing_event_handler.py @@ -13,9 +13,11 @@ LOG = logging.getLogger(__name__) class ProcessManager(SupervisorMixin, SupervisorLogMixin): def __init__(self): - self.commands = {'start': self.start_process, - 'stop': self.stop_process, - 'restart': self.restart_process} + self.commands = { + 'start': self.start_process, + 'stop': self.stop_process, + 'restart': self.restart_process + } def __call__(self, command, process_name): return self.commands[command](process_name) @@ -25,9 +27,9 @@ class ProcessManagingEventHandler(EventHandlerBase): """ Event handler that can manage processes managed by supervisor. - This EventHandler accepts messages that have a data["command"] key specifying + This EventHandler accepts messages that have a data['command'] key specifying a command to be executed. - Every message must contain a data["process_name"] field with the name of the + Every message must contain a data['process_name'] field with the name of the process to manage. This is the name specified in supervisor config files like so: [program:someprogram] @@ -50,8 +52,14 @@ class ProcessManagingEventHandler(EventHandlerBase): except SupervisorFault as fault: message['data']['error'] = fault.faultString finally: - message['data']['stdout'] = self.processmanager.read_stdout(data['process_name'], self.log_tail) - message['data']['stderr'] = self.processmanager.read_stderr(data['process_name'], self.log_tail) + message['data']['stdout'] = self.processmanager.read_stdout( + data['process_name'], + self.log_tail + ) + message['data']['stderr'] = self.processmanager.read_stderr( + data['process_name'], + self.log_tail + ) return message except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/lib/tfw/components/terminado_mini_server.py b/lib/tfw/components/terminado_mini_server.py index 9c79b83..3726097 100644 --- a/lib/tfw/components/terminado_mini_server.py +++ b/lib/tfw/components/terminado_mini_server.py @@ -14,15 +14,15 @@ LOG = logging.getLogger(__name__) class TerminadoMiniServer: def __init__(self, url, port, workdir, shellcmd): self.port = port - self._term_manager = SingleTermManager(shell_command=shellcmd, - term_settings={'cwd': workdir}) - self.application = Application( - [( - url, - TerminadoMiniServer.ResetterTermSocket, - {'term_manager': self._term_manager} - )] + self._term_manager = SingleTermManager( + shell_command=shellcmd, + term_settings={'cwd': workdir} ) + self.application = Application([( + url, + TerminadoMiniServer.ResetterTermSocket, + {'term_manager': self._term_manager} + )]) @property def term_manager(self): @@ -46,5 +46,10 @@ class TerminadoMiniServer: if __name__ == '__main__': LOG.info('Terminado Mini Server listening on %s', TFWENV.TERMINADO_PORT) - TerminadoMiniServer('/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, ['bash']).listen() + TerminadoMiniServer( + '/terminal', + TFWENV.TERMINADO_PORT, + TFWENV.TERMINADO_WD, + ['bash'] + ).listen() IOLoop.instance().start() diff --git a/lib/tfw/components/terminal_commands.py b/lib/tfw/components/terminal_commands.py index edaa16f..a47852d 100644 --- a/lib/tfw/components/terminal_commands.py +++ b/lib/tfw/components/terminal_commands.py @@ -19,13 +19,13 @@ class TerminalCommands(ABC): To receive events you need to subscribe TerminalCommand.callback to a HistoryMonitor instance. - Inherit from this class and define methods which start with "command_". When the user + Inherit from this class and define methods which start with "command\_". When the user executes the command specified after the underscore, your method will be invoked. All - such commands must expect the parameter *args which will contain the arguments of the + such commands must expect the parameter \*args which will contain the arguments of the command. For example to define a method that runs when someone starts vim in the terminal - you have to define a method like: "def command_vim(self, *args)" + you have to define a method like: "def command_vim(self, \*args)" You can also use this class to create new commands similarly. """ @@ -36,8 +36,12 @@ class TerminalCommands(ABC): self._setup_bashrc_aliases(bashrc) def _build_command_to_implementation_dict(self): - return {self._parse_command_name(fun): getattr(self, fun) for fun in dir(self) - if callable(getattr(self, fun)) and self._is_command_implementation(fun)} + return { + self._parse_command_name(fun): getattr(self, fun) + for fun in dir(self) + if callable(getattr(self, fun)) + and self._is_command_implementation(fun) + } def _setup_bashrc_aliases(self, bashrc): with open(bashrc, 'a') as ofile: diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/components/terminal_event_handler.py index 78041ef..ea135b6 100644 --- a/lib/tfw/components/terminal_event_handler.py +++ b/lib/tfw/components/terminal_event_handler.py @@ -16,9 +16,9 @@ class TerminalEventHandler(EventHandlerBase): sessions to connect to. You need to instanciate this in order for frontend terminals to work. - This EventHandler accepts messages that have a data["command"] key specifying + This EventHandler accepts messages that have a data['command'] key specifying a command to be executed. - The API of each command is documented in their respective handlers. + The API of each command is documented in their respective handler. """ def __init__(self, key, monitor): """ @@ -29,9 +29,19 @@ class TerminalEventHandler(EventHandlerBase): self.working_directory = TFWENV.TERMINADO_DIR self._historymonitor = monitor bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash'] - self.terminado_server = TerminadoMiniServer('/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, bash_as_user_cmd) - self.commands = {'write': self.write, - 'read': self.read} + + self.terminado_server = TerminadoMiniServer( + '/terminal', + TFWENV.TERMINADO_PORT, + TFWENV.TERMINADO_WD, + bash_as_user_cmd + ) + + self.commands = { + 'write': self.write, + 'read': self.read + } + if self._historymonitor: self._historymonitor.watch() self.terminado_server.listen() @@ -54,18 +64,18 @@ class TerminalEventHandler(EventHandlerBase): Writes a string to the terminal session (on the pty level). Useful for pre-typing and executing commands for the user. - :param data: TFW message data containing keys: - |-shellcmd: command to be written to the pty + :param data: TFW message data containing 'value' + (command to be written to the pty) """ - self.terminado_server.pty.write(data['shellcmd']) + self.terminado_server.pty.write(data['value']) def read(self, data): """ Reads the history of commands executed. - :param data: TFW message data containing keys: - |-count: the number of history elements to return - :return: message with list of commands in data['history'] + :param data: TFW message data containing 'count' + (the number of history elements to return) + :return dict: message with list of commands in data['history'] """ data['count'] = int(data.get('count', 1)) if self.historymonitor: diff --git a/lib/tfw/config/envvars.py b/lib/tfw/config/envvars.py index f239d5c..ef590c6 100644 --- a/lib/tfw/config/envvars.py +++ b/lib/tfw/config/envvars.py @@ -1,6 +1,6 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from envvars import prefixed_envvars_to_namedtuple +from envvars import LazyEnvironment -TFWENV = prefixed_envvars_to_namedtuple('TFW_', 'tfwenvtuple') +TFWENV = LazyEnvironment('TFW_', 'tfwenvtuple').environment diff --git a/lib/tfw/decorators/__init__.py b/lib/tfw/decorators/__init__.py index 80c5788..ed79d7f 100644 --- a/lib/tfw/decorators/__init__.py +++ b/lib/tfw/decorators/__init__.py @@ -2,3 +2,4 @@ # All Rights Reserved. See LICENSE file for details. from .rate_limiter import RateLimiter +from .lazy_property import lazy_property diff --git a/lib/tfw/decorators/lazy_property.py b/lib/tfw/decorators/lazy_property.py new file mode 100644 index 0000000..80ddbeb --- /dev/null +++ b/lib/tfw/decorators/lazy_property.py @@ -0,0 +1,19 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + + +class lazy_property: + """ + Decorator that replaces a function with the value + it calculates on the first call. + """ + def __init__(self, func): + self.func = func + self.__doc__ = func.__doc__ + + def __get__(self, instance, owner): + if instance is None: + return self # avoids potential __new__ TypeError + value = self.func(instance) + setattr(instance, self.func.__name__, value) + return value diff --git a/lib/tfw/fsm_base.py b/lib/tfw/fsm_base.py index d2e1df0..7296627 100644 --- a/lib/tfw/fsm_base.py +++ b/lib/tfw/fsm_base.py @@ -19,15 +19,16 @@ class FSMBase(CallbackMixin): states, transitions = [], [] def __init__(self, initial: str = None, accepted_states: List[str] = None): - CallbackMixin.__init__(self) self.accepted_states = accepted_states or [self.states[-1]] - self.machine = Machine(model=self, - states=self.states, - transitions=self.transitions, - initial=initial or self.states[0], - send_event=True, - ignore_invalid_triggers=True, - after_state_change='execute_callbacks') + self.machine = Machine( + model=self, + states=self.states, + transitions=self.transitions, + initial=initial or self.states[0], + send_event=True, + ignore_invalid_triggers=True, + after_state_change='execute_callbacks' + ) def execute_callbacks(self, event_data): self._execute_callbacks(event_data.kwargs) diff --git a/lib/tfw/linear_fsm.py b/lib/tfw/linear_fsm.py index 2728655..1f0e3af 100644 --- a/lib/tfw/linear_fsm.py +++ b/lib/tfw/linear_fsm.py @@ -11,12 +11,19 @@ class LinearFSM(FSMBase): a number of steps specified in the constructor. It automatically sets up 2 actions (triggers) between states as such: (0) -- step_1 --> (1) -- step_2 --> (2) -- step_3 --> (3) ... and so on - \-step_next-/ \-step_next-/ \-step_next-/ """ def __init__(self, number_of_steps): self.states = list(map(str, range(number_of_steps))) self.transitions = [] for index in self.states[:-1]: - self.transitions.append({'trigger': f'step_{int(index)+1}', 'source': index, 'dest': str(int(index)+1)}) - self.transitions.append({'trigger': 'step_next', 'source': index, 'dest': str(int(index)+1)}) + self.transitions.append({ + 'trigger': f'step_{int(index)+1}', + 'source': index, + 'dest': str(int(index)+1) + }) + self.transitions.append({ + 'trigger': 'step_next', + 'source': index, + 'dest': str(int(index)+1) + }) super(LinearFSM, self).__init__() diff --git a/lib/tfw/mixins/callback_mixin.py b/lib/tfw/mixins/callback_mixin.py index e5cd534..3c94e71 100644 --- a/lib/tfw/mixins/callback_mixin.py +++ b/lib/tfw/mixins/callback_mixin.py @@ -3,17 +3,20 @@ from functools import partial +from tfw.decorators import lazy_property + class CallbackMixin: - def __init__(self): - self._callbacks = [] + @lazy_property + def _callbacks(self): + return [] def subscribe_callback(self, callback, *args, **kwargs): """ Subscribe a callable to invoke once an event is triggered. :param callback: callable to be executed on events - :param *args: arguments passed to callable - :param **kwargs: kwargs passed to callable + :param args: arguments passed to callable + :param kwargs: kwargs passed to callable """ fun = partial(callback, *args, **kwargs) self._callbacks.append(fun) diff --git a/lib/tfw/mixins/observer_mixin.py b/lib/tfw/mixins/observer_mixin.py index 4101d96..712a31e 100644 --- a/lib/tfw/mixins/observer_mixin.py +++ b/lib/tfw/mixins/observer_mixin.py @@ -3,10 +3,13 @@ from watchdog.observers import Observer +from tfw.decorators import lazy_property + class ObserverMixin: - def __init__(self): - self.observer = Observer() + @lazy_property + def observer(self): + return Observer() def watch(self): self.observer.start() diff --git a/lib/tfw/mixins/supervisor_mixin.py b/lib/tfw/mixins/supervisor_mixin.py index 9360395..00cf398 100644 --- a/lib/tfw/mixins/supervisor_mixin.py +++ b/lib/tfw/mixins/supervisor_mixin.py @@ -6,21 +6,14 @@ from xmlrpc.client import Fault as SupervisorFault from contextlib import suppress from os import remove +from tfw.decorators import lazy_property from tfw.config import TFWENV -def get_supervisor_instance(): - return xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor - - class SupervisorBaseMixin: - supervisor = get_supervisor_instance() - - def acquire_own_supervisor_instance(self): - """ - Give this instance non-static, local xmlrpc client - """ - self.supervisor = get_supervisor_instance() + @lazy_property + def supervisor(self): + return xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor class SupervisorMixin(SupervisorBaseMixin): diff --git a/lib/tfw/networking/message_sender.py b/lib/tfw/networking/message_sender.py index d1bf0d5..2210d0a 100644 --- a/lib/tfw/networking/message_sender.py +++ b/lib/tfw/networking/message_sender.py @@ -26,5 +26,7 @@ class MessageSender: 'timestamp': datetime.now().isoformat(), 'message': message } - self.server_connector.send({'key': self.key, - 'data': data}) + self.server_connector.send({ + 'key': self.key, + 'data': data + }) diff --git a/lib/tfw/networking/serialization.py b/lib/tfw/networking/serialization.py index 1eaf933..6a6a3e7 100644 --- a/lib/tfw/networking/serialization.py +++ b/lib/tfw/networking/serialization.py @@ -43,11 +43,17 @@ def deserialize_tfw_msg(*args): def _serialize_all(*args): - return tuple(_serialize_single(arg) for arg in args) + return tuple( + _serialize_single(arg) + for arg in args + ) def _deserialize_all(*args): - return tuple(_deserialize_single(arg) for arg in args) + return tuple( + _deserialize_single(arg) + for arg in args + ) def _serialize_single(data): diff --git a/lib/tfw/networking/server/tfw_server.py b/lib/tfw/networking/server/tfw_server.py index 89ad3d4..78cc8ee 100644 --- a/lib/tfw/networking/server/tfw_server.py +++ b/lib/tfw/networking/server/tfw_server.py @@ -30,13 +30,16 @@ class TFWServer: self._fsm.subscribe_callback(self._fsm_updater.update) self._event_handler_connector = EventHandlerConnector() - self.application = Application( - [(r'/ws', ZMQWebSocketProxy, {'make_eventhandler_message': self.make_eventhandler_message, - 'proxy_filter': self.proxy_filter, - 'handle_trigger': self.handle_trigger, - 'event_handler_connector': self._event_handler_connector})] + self.application = Application([( + r'/ws', ZMQWebSocketProxy,{ + 'make_eventhandler_message': self.make_eventhandler_message, + 'proxy_filter': self.proxy_filter, + 'handle_trigger': self.handle_trigger, + 'event_handler_connector': self._event_handler_connector + })] ) - #self.controller_responder = ControllerResponder(self.fsm) TODO: add this once controller stuff is resolved + # self.controller_responder = ControllerResponder(self.fsm) + # TODO: add this once controller stuff is resolved @property def fsm(self): @@ -97,8 +100,11 @@ class FSMManager: self.trigger_predicates[trigger].extend(predicates) def unsubscribe_predicate(self, trigger, *predicates): - self.trigger_predicates[trigger] = [predicate for predicate in self.trigger_predicates[trigger] - not in predicates] + self.trigger_predicates[trigger] = [ + predicate + for predicate in self.trigger_predicates[trigger] + not in predicates + ] class FSMUpdater: @@ -111,10 +117,18 @@ class FSMUpdater: self.uplink.send(self.generate_fsm_update()) def generate_fsm_update(self): - return {'key': 'FSMUpdate', - 'data': self.get_fsm_state_and_transitions()} + return { + 'key': 'FSMUpdate', + 'data': self.get_fsm_state_and_transitions() + } def get_fsm_state_and_transitions(self): state = self.fsm.state - valid_transitions = [{'trigger': trigger} for trigger in self.fsm.machine.get_triggers(self.fsm.state)] - return {'current_state': state, 'valid_transitions': valid_transitions} + valid_transitions = [ + {'trigger': trigger} + for trigger in self.fsm.machine.get_triggers(self.fsm.state) + ] + return { + 'current_state': state, + 'valid_transitions': valid_transitions + }