From b68ceeb394440d419f6f286c397429eba12bbbef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 31 May 2018 13:53:11 +0200 Subject: [PATCH 01/18] Rework LogMonitoringEH API to adhere new conventions --- lib/tfw/components/log_monitoring_event_handler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/tfw/components/log_monitoring_event_handler.py b/lib/tfw/components/log_monitoring_event_handler.py index e28a3ad..e5687d0 100644 --- a/lib/tfw/components/log_monitoring_event_handler.py +++ b/lib/tfw/components/log_monitoring_event_handler.py @@ -1,7 +1,6 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. - from tfw import EventHandlerBase from tfw.mixins import MonitorManagerMixin from tfw.config.logs import logging @@ -31,7 +30,7 @@ class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) def handle_process_name(self, data): - self.set_monitor_args(data['process_name'], self.log_tail) + self.set_monitor_args(data['value'], self.log_tail) def handle_log_tail(self, data): - self.set_monitor_args(self.process_name, data['log_tail']) + self.set_monitor_args(self.process_name, data['value']) From ea251ef474e960f26e038aa1a51ae35e0c5f658e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 31 May 2018 14:03:11 +0200 Subject: [PATCH 02/18] Document LogMonitoringEventHandler --- .../log_monitoring_event_handler.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/tfw/components/log_monitoring_event_handler.py b/lib/tfw/components/log_monitoring_event_handler.py index e5687d0..078e2e8 100644 --- a/lib/tfw/components/log_monitoring_event_handler.py +++ b/lib/tfw/components/log_monitoring_event_handler.py @@ -10,6 +10,12 @@ LOG = logging.getLogger(__name__) class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): + """ + Monitors the output of a supervisor process (stdout, stderr) and + sends the results to the frontend. + + Exposes API to change monitoring parameters. + """ def __init__(self, key, process_name, log_tail=0): super().__init__(key) self.process_name = process_name @@ -30,7 +36,21 @@ class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) def handle_process_name(self, data): + """ + Changes the monitored process. + + :param data: TFW message data containing keys: + |-value: name of the process to monitor + """ self.set_monitor_args(data['value'], self.log_tail) def handle_log_tail(self, data): + """ + Sets tail length of the log the monitor will send + to the frontend (the monitor will send back the last + 'value' characters of the log). + + :param data: TFW message data containing keys: + |-value: new tail length + """ self.set_monitor_args(self.process_name, data['value']) From e98c41d3cfe8666149f8a16fec1114a1366890bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 31 May 2018 14:08:29 +0200 Subject: [PATCH 03/18] Comply new API style in TerminalEH --- lib/tfw/components/terminal_event_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/components/terminal_event_handler.py index 78041ef..9f7a640 100644 --- a/lib/tfw/components/terminal_event_handler.py +++ b/lib/tfw/components/terminal_event_handler.py @@ -55,9 +55,9 @@ class TerminalEventHandler(EventHandlerBase): Useful for pre-typing and executing commands for the user. :param data: TFW message data containing keys: - |-shellcmd: command to be written to the pty + |-value: command to be written to the pty """ - self.terminado_server.pty.write(data['shellcmd']) + self.terminado_server.pty.write(data['value']) def read(self, data): """ From 06c2fc97ad7cc6a59168e5995894167100a4cf6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 1 Jun 2018 13:58:50 +0200 Subject: [PATCH 04/18] Implement decorator to lazy initialise a property --- lib/tfw/decorators/__init__.py | 1 + lib/tfw/decorators/lazy_initialise.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 lib/tfw/decorators/lazy_initialise.py diff --git a/lib/tfw/decorators/__init__.py b/lib/tfw/decorators/__init__.py index 80c5788..cd61318 100644 --- a/lib/tfw/decorators/__init__.py +++ b/lib/tfw/decorators/__init__.py @@ -2,3 +2,4 @@ # All Rights Reserved. See LICENSE file for details. from .rate_limiter import RateLimiter +from .lazy_initialise import LazyInitialise diff --git a/lib/tfw/decorators/lazy_initialise.py b/lib/tfw/decorators/lazy_initialise.py new file mode 100644 index 0000000..615721b --- /dev/null +++ b/lib/tfw/decorators/lazy_initialise.py @@ -0,0 +1,19 @@ +# Copyright (C) 2018 Avatao.com Innovative Learning Kft. +# All Rights Reserved. See LICENSE file for details. + + +class LazyInitialise: + """ + Decorator that replaces a function with the value + it calculates on the first call. + """ + def __init__(self, func): + self.func = func + self.__doc__ = func.__doc__ + + def __get__(self, instance, owner): + if instance is None: + raise TypeError('Cannot get object property from class!') + value = self.func(instance) + setattr(instance, self.func.__name__, value) + return value From 8817f991a59d0071530b094cb8ba01af64ebd3f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 1 Jun 2018 14:00:44 +0200 Subject: [PATCH 05/18] Use power of LazyInitialise to murder TFW/TAOENV global state --- lib/envvars/__init__.py | 21 ++++++++++++++++----- lib/tao/config/envvars.py | 4 ++-- lib/tfw/config/envvars.py | 4 ++-- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/lib/envvars/__init__.py b/lib/envvars/__init__.py index 712a926..11faa64 100644 --- a/lib/envvars/__init__.py +++ b/lib/envvars/__init__.py @@ -4,9 +4,20 @@ from collections import namedtuple from os import environ +from tfw.decorators import LazyInitialise -def prefixed_envvars_to_namedtuple(prefix: str, tuple_name: str): - envvars = {envvar.replace(prefix, '', 1): environ.get(envvar) - for envvar in environ.keys() - if envvar.startswith(prefix)} - return namedtuple(tuple_name, envvars)(**envvars) + +class LazyEnvironment: + def __init__(self, prefix, tuple_name): + self._prefix = prefix + self._tuple_name = tuple_name + + @LazyInitialise + def environment(self): + return self.prefixed_envvars_to_namedtuple() + + def prefixed_envvars_to_namedtuple(self): + envvars = {envvar.replace(self._prefix, '', 1): environ.get(envvar) + for envvar in environ.keys() + if envvar.startswith(self._prefix)} + return namedtuple(self._tuple_name, envvars)(**envvars) diff --git a/lib/tao/config/envvars.py b/lib/tao/config/envvars.py index da46222..260d057 100644 --- a/lib/tao/config/envvars.py +++ b/lib/tao/config/envvars.py @@ -1,6 +1,6 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from envvars import prefixed_envvars_to_namedtuple +from envvars import LazyEnvironment -TAOENV = prefixed_envvars_to_namedtuple('AVATAO_', 'taoenvtuple') +TAOENV = LazyEnvironment('AVATAO_', 'taoenvtuple').environment diff --git a/lib/tfw/config/envvars.py b/lib/tfw/config/envvars.py index f239d5c..ef590c6 100644 --- a/lib/tfw/config/envvars.py +++ b/lib/tfw/config/envvars.py @@ -1,6 +1,6 @@ # Copyright (C) 2018 Avatao.com Innovative Learning Kft. # All Rights Reserved. See LICENSE file for details. -from envvars import prefixed_envvars_to_namedtuple +from envvars import LazyEnvironment -TFWENV = prefixed_envvars_to_namedtuple('TFW_', 'tfwenvtuple') +TFWENV = LazyEnvironment('TFW_', 'tfwenvtuple').environment From 9f3a3b501f2fffbecbbe1705b6ff18d3193a8526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 1 Jun 2018 14:01:32 +0200 Subject: [PATCH 06/18] Initialise supervisor xmlrpc client lazily to avoid globals --- lib/tfw/components/log_monitor.py | 1 - lib/tfw/mixins/supervisor_mixin.py | 15 ++++----------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/lib/tfw/components/log_monitor.py b/lib/tfw/components/log_monitor.py index 287618f..d7672d1 100644 --- a/lib/tfw/components/log_monitor.py +++ b/lib/tfw/components/log_monitor.py @@ -29,7 +29,6 @@ class LogMonitor(ObserverMixin): class SendLogWatchdogEventHandler(PatternMatchingWatchdogEventHandler, SupervisorLogMixin): def __init__(self, process_name, log_tail=0): - self.acquire_own_supervisor_instance() # This thread-localises the xmlrpc client self.process_name = process_name self.procinfo = self.supervisor.getProcessInfo(self.process_name) super().__init__([self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']]) diff --git a/lib/tfw/mixins/supervisor_mixin.py b/lib/tfw/mixins/supervisor_mixin.py index 9360395..bdd43ea 100644 --- a/lib/tfw/mixins/supervisor_mixin.py +++ b/lib/tfw/mixins/supervisor_mixin.py @@ -6,21 +6,14 @@ from xmlrpc.client import Fault as SupervisorFault from contextlib import suppress from os import remove +from tfw.decorators import LazyInitialise from tfw.config import TFWENV -def get_supervisor_instance(): - return xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor - - class SupervisorBaseMixin: - supervisor = get_supervisor_instance() - - def acquire_own_supervisor_instance(self): - """ - Give this instance non-static, local xmlrpc client - """ - self.supervisor = get_supervisor_instance() + @LazyInitialise + def supervisor(self): + return xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor class SupervisorMixin(SupervisorBaseMixin): From cebacb15e6d6c1d67a860876392124f4c7b924de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 1 Jun 2018 14:06:40 +0200 Subject: [PATCH 07/18] Use power of LazyInitialise to replace ObserverMixin.__init__() --- lib/tfw/components/directory_monitor.py | 1 - lib/tfw/components/history_monitor.py | 1 - lib/tfw/components/log_monitor.py | 1 - lib/tfw/mixins/observer_mixin.py | 7 +++++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index ba423be..3ad6732 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -16,7 +16,6 @@ LOG = logging.getLogger(__name__) class DirectoryMonitor(ObserverMixin): def __init__(self, directories): - ObserverMixin.__init__(self) self.eventhandler = IdeReloadWatchdogEventHandler() for directory in directories: self.observer.schedule(self.eventhandler, directory, recursive=True) diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index d5586a7..67bfc3b 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -37,7 +37,6 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC): """ def __init__(self, histfile): CallbackMixin.__init__(self) - ObserverMixin.__init__(self) self.histfile = histfile self._history = [] self._last_length = len(self._history) diff --git a/lib/tfw/components/log_monitor.py b/lib/tfw/components/log_monitor.py index d7672d1..d8d1825 100644 --- a/lib/tfw/components/log_monitor.py +++ b/lib/tfw/components/log_monitor.py @@ -14,7 +14,6 @@ from tfw.mixins import ObserverMixin, SupervisorLogMixin class LogMonitor(ObserverMixin): def __init__(self, process_name, log_tail=0): self.prevent_log_recursion() - ObserverMixin.__init__(self) event_handler = SendLogWatchdogEventHandler(process_name, log_tail=log_tail) self.observer.schedule( event_handler, diff --git a/lib/tfw/mixins/observer_mixin.py b/lib/tfw/mixins/observer_mixin.py index 4101d96..198c405 100644 --- a/lib/tfw/mixins/observer_mixin.py +++ b/lib/tfw/mixins/observer_mixin.py @@ -3,10 +3,13 @@ from watchdog.observers import Observer +from tfw.decorators import LazyInitialise + class ObserverMixin: - def __init__(self): - self.observer = Observer() + @LazyInitialise + def observer(self): + return Observer() def watch(self): self.observer.start() From 6f3db18146cf168c01836f57f05e633cb1a750a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 1 Jun 2018 14:21:32 +0200 Subject: [PATCH 08/18] Use power of LazyInitialise to replace CallbackMixin.__init__() --- lib/tfw/components/history_monitor.py | 1 - lib/tfw/fsm_base.py | 1 - lib/tfw/mixins/callback_mixin.py | 7 +++++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index 67bfc3b..b103bb9 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -36,7 +36,6 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC): See examples below. """ def __init__(self, histfile): - CallbackMixin.__init__(self) self.histfile = histfile self._history = [] self._last_length = len(self._history) diff --git a/lib/tfw/fsm_base.py b/lib/tfw/fsm_base.py index d2e1df0..85ac12f 100644 --- a/lib/tfw/fsm_base.py +++ b/lib/tfw/fsm_base.py @@ -19,7 +19,6 @@ class FSMBase(CallbackMixin): states, transitions = [], [] def __init__(self, initial: str = None, accepted_states: List[str] = None): - CallbackMixin.__init__(self) self.accepted_states = accepted_states or [self.states[-1]] self.machine = Machine(model=self, states=self.states, diff --git a/lib/tfw/mixins/callback_mixin.py b/lib/tfw/mixins/callback_mixin.py index e5cd534..971de5c 100644 --- a/lib/tfw/mixins/callback_mixin.py +++ b/lib/tfw/mixins/callback_mixin.py @@ -3,10 +3,13 @@ from functools import partial +from tfw.decorators import LazyInitialise + class CallbackMixin: - def __init__(self): - self._callbacks = [] + @LazyInitialise + def _callbacks(self): + return [] def subscribe_callback(self, callback, *args, **kwargs): """ From 469c1e7217762f3ede95af20854cc805bf01b6b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 1 Jun 2018 14:24:00 +0200 Subject: [PATCH 09/18] Name new TFW version due to API breaking --- .drone.yml | 2 +- VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 9e21f9e..0b24068 100644 --- a/.drone.yml +++ b/.drone.yml @@ -10,4 +10,4 @@ pipeline: - docker push eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG} when: event: 'tag' - branch: refs/tags/egyptianmau-20* + branch: refs/tags/bombay-20* diff --git a/VERSION b/VERSION index 23d699d..4958238 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -egyptianmau +bombay From c0fb28c46c1190147c54a8735197b79f985fcb36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 1 Jun 2018 15:15:07 +0200 Subject: [PATCH 10/18] Stop LogMonitor on EventHandler.cleanup() in LogMonitoringEH --- lib/tfw/components/log_monitoring_event_handler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/tfw/components/log_monitoring_event_handler.py b/lib/tfw/components/log_monitoring_event_handler.py index 078e2e8..10058ce 100644 --- a/lib/tfw/components/log_monitoring_event_handler.py +++ b/lib/tfw/components/log_monitoring_event_handler.py @@ -54,3 +54,6 @@ class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): |-value: new tail length """ self.set_monitor_args(self.process_name, data['value']) + + def cleanup(self): + self.monitor.stop() From e80cce00f391a88f92a410c9c368b3700d4aaf1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 1 Jun 2018 16:20:20 +0200 Subject: [PATCH 11/18] Add sphinx API documentation --- docs/Makefile | 20 +++ docs/make.bat | 36 ++++ docs/source/components/components.rst | 25 +++ docs/source/conf.py | 166 ++++++++++++++++++ docs/source/foundations/eventhandlers.rst | 12 ++ docs/source/foundations/fsms.rst | 12 ++ docs/source/foundations/tfwserver.rst | 7 + docs/source/index.rst | 45 +++++ docs/source/networking/networking.rst | 17 ++ lib/tfw/components/history_monitor.py | 8 +- lib/tfw/components/ide_event_handler.py | 31 ++-- .../log_monitoring_event_handler.py | 13 +- .../process_managing_event_handler.py | 4 +- lib/tfw/components/terminal_commands.py | 6 +- lib/tfw/components/terminal_event_handler.py | 14 +- lib/tfw/linear_fsm.py | 1 - 16 files changed, 381 insertions(+), 36 deletions(-) create mode 100644 docs/Makefile create mode 100644 docs/make.bat create mode 100644 docs/source/components/components.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/foundations/eventhandlers.rst create mode 100644 docs/source/foundations/fsms.rst create mode 100644 docs/source/foundations/tfwserver.rst create mode 100644 docs/source/index.rst create mode 100644 docs/source/networking/networking.rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..1a264d7 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = baseimage-tutorial-framework +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..0a0210b --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build +set SPHINXPROJ=baseimage-tutorial-framework + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/source/components/components.rst b/docs/source/components/components.rst new file mode 100644 index 0000000..87dee5b --- /dev/null +++ b/docs/source/components/components.rst @@ -0,0 +1,25 @@ +Components +---------- + +.. automodule:: tfw.components + +.. autoclass:: IdeEventHandler + :members: + +.. autoclass:: TerminalEventHandler + :members: + +.. autoclass:: ProcessManagingEventHandler + :members: + +.. autoclass:: LogMonitoringEventHandler + :members: + +.. autoclass:: TerminalCommands + :members: + +.. autoclass:: HistoryMonitor + :members: + +.. autoclass:: BashMonitor + :members: diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..5c596d0 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../../lib')) + + +# -- Project information ----------------------------------------------------- + +project = 'baseimage-tutorial-framework' +copyright = '2018, Avatao Innovative Learning Kft' +author = 'Kristóf Tóth' + +# The short X.Y version +version = '' +# The full version, including alpha/beta/rc tags +release = 'bombay' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'baseimage-tutorial-frameworkdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'baseimage-tutorial-framework.tex', 'baseimage-tutorial-framework Documentation', + 'Kristóf Tóth', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'baseimage-tutorial-framework', 'baseimage-tutorial-framework Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'baseimage-tutorial-framework', 'baseimage-tutorial-framework Documentation', + author, 'baseimage-tutorial-framework', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Extension configuration ------------------------------------------------- +def skip(app, what, name, obj, skip, options): + if name == "__init__": + return False + return skip + +def setup(app): + app.connect("autodoc-skip-member", skip) diff --git a/docs/source/foundations/eventhandlers.rst b/docs/source/foundations/eventhandlers.rst new file mode 100644 index 0000000..9b8d91c --- /dev/null +++ b/docs/source/foundations/eventhandlers.rst @@ -0,0 +1,12 @@ +Event handler base classes +-------------------------- + +Subclass these to create your cusom event handlers. + +.. automodule:: tfw + +.. autoclass:: EventHandlerBase + :members: + +.. autoclass:: TriggeredEventHandler + :members: diff --git a/docs/source/foundations/fsms.rst b/docs/source/foundations/fsms.rst new file mode 100644 index 0000000..190ee8b --- /dev/null +++ b/docs/source/foundations/fsms.rst @@ -0,0 +1,12 @@ +FSM base classes +---------------- + +Subclass these to create an FSM that fits your tutorial/challenge. + +.. automodule:: tfw + +.. autoclass:: FSMBase + :members: + +.. autoclass:: LinearFSM + :members: diff --git a/docs/source/foundations/tfwserver.rst b/docs/source/foundations/tfwserver.rst new file mode 100644 index 0000000..3dddaa4 --- /dev/null +++ b/docs/source/foundations/tfwserver.rst @@ -0,0 +1,7 @@ +TFWServer +--------- + +.. automodule:: tfw.networking + +.. autoclass:: TFWServer + :members: diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..3d674ad --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,45 @@ +.. baseimage-tutorial-framework documentation master file, created by + sphinx-quickstart on Fri Jun 1 14:29:07 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to baseimage-tutorial-framework's documentation! +======================================================== + +Foundations +----------- + +This part covers the soil which the framework is based on and stuff you will need to develop your own challenges. + +.. toctree:: + :glob: + + foundations/* + +Networking +---------- + +You can use these to send messages to the frontend or the event handlers through TFWServer. + +.. toctree:: + :glob: + + networking/* + +Components +---------- + +These are pre-written components for you to use, such as our IDE, terminal or console. + +.. toctree:: + :glob: + + components/* + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/networking/networking.rst b/docs/source/networking/networking.rst new file mode 100644 index 0000000..317d266 --- /dev/null +++ b/docs/source/networking/networking.rst @@ -0,0 +1,17 @@ +Networking +---------- + +.. automodule:: tfw.networking + +.. autoclass:: TFWServerConnector + :members: + +.. automodule:: tfw.networking.event_handlers + +.. autoclass:: ServerUplinkConnector + :members: + +.. automodule:: tfw.networking + +.. autoclass:: MessageSender + :members: diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index b103bb9..e775c08 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -74,10 +74,10 @@ class BashMonitor(HistoryMonitor): HistoryMonitor for monitoring bash CLI sessions. This requires the following to be set in bash (note that this is done automatically by TFW): - PROMPT_COMMAND="history -a" - shopt -s cmdhist - shopt -s histappend - unset HISTCONTROL + PROMPT_COMMAND="history -a" + shopt -s cmdhist + shopt -s histappend + unset HISTCONTROL """ @property def command_pattern(self): diff --git a/lib/tfw/components/ide_event_handler.py b/lib/tfw/components/ide_event_handler.py index 7c9fda3..4c4fa6f 100644 --- a/lib/tfw/components/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler.py @@ -101,9 +101,10 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): When any file in the selected directory changes they are automatically refreshed on the frontend (this is done by listening to inotify events). - This EventHandler accepts messages that have a data["command"] key specifying + This EventHandler accepts messages that have a data['command'] key specifying a command to be executed. - The API of each command is documented in their respective handlers. + + The API of each command is documented in their respective handler. """ def __init__(self, key, directory, allowed_directories, selected_file=None, exclude=None, additional_watched_directories=None): @@ -112,7 +113,7 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): :param directory: working directory which the EventHandler should serve files from :param allowed_directories: list of directories that can be switched to using the selectdir command :param selected_file: file that is selected by default - :param exclude: list of filenames that should not appear between files (for *.o, *.pyc, etc.) + :param exclude: list of filenames that should not appear between files (for .o, .pyc, etc.) :param additional_watched_directories: refresh the selected file when files change in these directories (the working directory is watched by default, this is useful for symlinks and such) @@ -139,7 +140,7 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): """ Read the currently selected file. - :return: message with the contents of the file in data['content'] + :return dict: message with the contents of the file in data['content'] """ try: data['content'] = self.filemanager.file_contents @@ -155,8 +156,9 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): """ Overwrites a file with the desired string. - :param data: TFW message data containing keys: - |-string: containing the desired file contents + :param data: TFW message data containing key 'content' + (new file content) + """ self.monitor.ignore = self.monitor.ignore + 1 try: @@ -170,8 +172,8 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): """ Selects a file from the current directory. - :param data: TFW message data containing keys: - |-filename: name of file to select relative to the current directory + :param data: TFW message data containing 'filename' + (name of file to select relative to the current directory) """ try: self.filemanager.filename = data['filename'] @@ -183,10 +185,10 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): """ Select a new working directory to display files from. - :param data: TFW message data containing keys: - |-directory: absolute path of diretory to select. - must be a path whitelisted in - self.allowed_directories + :param data: TFW message data containing 'directory' + (absolute path of diretory to select. + must be a path whitelisted in + self.allowed_directories) """ try: self.filemanager.workdir = data['directory'] @@ -204,8 +206,9 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): """ Overwrite list of excluded files - :param data: TFW message data containing keys: - |-exclude: list of filename patterns to be excluded, e.g.: ["*.pyc", "*.o"] + :param data: TFW message data containing 'exclude' + (list of unix-style filename patterns to be excluded, + e.g.: ["\*.pyc", "\*.o") """ try: self.filemanager.exclude = list(data['exclude']) diff --git a/lib/tfw/components/log_monitoring_event_handler.py b/lib/tfw/components/log_monitoring_event_handler.py index 10058ce..eb9782f 100644 --- a/lib/tfw/components/log_monitoring_event_handler.py +++ b/lib/tfw/components/log_monitoring_event_handler.py @@ -14,7 +14,10 @@ class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): Monitors the output of a supervisor process (stdout, stderr) and sends the results to the frontend. - Exposes API to change monitoring parameters. + Accepts messages that have a data['command'] key specifying + a command to be executed. + + The API of each command is documented in their respective handler. """ def __init__(self, key, process_name, log_tail=0): super().__init__(key) @@ -39,8 +42,8 @@ class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): """ Changes the monitored process. - :param data: TFW message data containing keys: - |-value: name of the process to monitor + :param data: TFW message data containing 'value' + (name of the process to monitor) """ self.set_monitor_args(data['value'], self.log_tail) @@ -50,8 +53,8 @@ class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): to the frontend (the monitor will send back the last 'value' characters of the log). - :param data: TFW message data containing keys: - |-value: new tail length + :param data: TFW message data containing 'value' + (new tail length) """ self.set_monitor_args(self.process_name, data['value']) diff --git a/lib/tfw/components/process_managing_event_handler.py b/lib/tfw/components/process_managing_event_handler.py index a512c1e..0a3d052 100644 --- a/lib/tfw/components/process_managing_event_handler.py +++ b/lib/tfw/components/process_managing_event_handler.py @@ -25,9 +25,9 @@ class ProcessManagingEventHandler(EventHandlerBase): """ Event handler that can manage processes managed by supervisor. - This EventHandler accepts messages that have a data["command"] key specifying + This EventHandler accepts messages that have a data['command'] key specifying a command to be executed. - Every message must contain a data["process_name"] field with the name of the + Every message must contain a data['process_name'] field with the name of the process to manage. This is the name specified in supervisor config files like so: [program:someprogram] diff --git a/lib/tfw/components/terminal_commands.py b/lib/tfw/components/terminal_commands.py index edaa16f..ae22975 100644 --- a/lib/tfw/components/terminal_commands.py +++ b/lib/tfw/components/terminal_commands.py @@ -19,13 +19,13 @@ class TerminalCommands(ABC): To receive events you need to subscribe TerminalCommand.callback to a HistoryMonitor instance. - Inherit from this class and define methods which start with "command_". When the user + Inherit from this class and define methods which start with "command\_". When the user executes the command specified after the underscore, your method will be invoked. All - such commands must expect the parameter *args which will contain the arguments of the + such commands must expect the parameter \*args which will contain the arguments of the command. For example to define a method that runs when someone starts vim in the terminal - you have to define a method like: "def command_vim(self, *args)" + you have to define a method like: "def command_vim(self, \*args)" You can also use this class to create new commands similarly. """ diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/components/terminal_event_handler.py index 9f7a640..3f7e208 100644 --- a/lib/tfw/components/terminal_event_handler.py +++ b/lib/tfw/components/terminal_event_handler.py @@ -16,9 +16,9 @@ class TerminalEventHandler(EventHandlerBase): sessions to connect to. You need to instanciate this in order for frontend terminals to work. - This EventHandler accepts messages that have a data["command"] key specifying + This EventHandler accepts messages that have a data['command'] key specifying a command to be executed. - The API of each command is documented in their respective handlers. + The API of each command is documented in their respective handler. """ def __init__(self, key, monitor): """ @@ -54,8 +54,8 @@ class TerminalEventHandler(EventHandlerBase): Writes a string to the terminal session (on the pty level). Useful for pre-typing and executing commands for the user. - :param data: TFW message data containing keys: - |-value: command to be written to the pty + :param data: TFW message data containing 'value' + (command to be written to the pty) """ self.terminado_server.pty.write(data['value']) @@ -63,9 +63,9 @@ class TerminalEventHandler(EventHandlerBase): """ Reads the history of commands executed. - :param data: TFW message data containing keys: - |-count: the number of history elements to return - :return: message with list of commands in data['history'] + :param data: TFW message data containing 'count' + (the number of history elements to return) + :return dict: message with list of commands in data['history'] """ data['count'] = int(data.get('count', 1)) if self.historymonitor: diff --git a/lib/tfw/linear_fsm.py b/lib/tfw/linear_fsm.py index 2728655..61e4c43 100644 --- a/lib/tfw/linear_fsm.py +++ b/lib/tfw/linear_fsm.py @@ -11,7 +11,6 @@ class LinearFSM(FSMBase): a number of steps specified in the constructor. It automatically sets up 2 actions (triggers) between states as such: (0) -- step_1 --> (1) -- step_2 --> (2) -- step_3 --> (3) ... and so on - \-step_next-/ \-step_next-/ \-step_next-/ """ def __init__(self, number_of_steps): self.states = list(map(str, range(number_of_steps))) From 7a67b68cbaa4330732b44c56a1d686ebcc098a9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 1 Jun 2018 16:23:44 +0200 Subject: [PATCH 12/18] Add info on Sphinx-docs to README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e3b7f54..01b6933 100644 --- a/README.md +++ b/README.md @@ -80,4 +80,6 @@ The TFW message format: Most of the components you need have docstrings included (hang on tight, this is work in progress) – refer to them for usage info. +In the `docs` folder you can find our Sphinx-based API documentation, which you can build using the included `Makefile` (you need to have Sphinx installed, please reach out to us if you have trouble building the docs). + To get started you should take a look at the [test-tutorial-framework](https://github.com/avatao-content/test-tutorial-framework) repository, which serves as an example project as well. From a743b01bcfcabc933eaaea97378d916f42568418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 1 Jun 2018 17:19:58 +0200 Subject: [PATCH 13/18] Reduce line length to 120 --- .pylintrc | 2 +- lib/tfw/components/terminal_event_handler.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.pylintrc b/.pylintrc index 7fb46c0..f36e237 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,5 +1,5 @@ [TYPECHECK] ignored-modules = zmq -max-line-length = 150 +max-line-length = 120 disable = missing-docstring, too-few-public-methods, invalid-name diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/components/terminal_event_handler.py index 3f7e208..5d1956e 100644 --- a/lib/tfw/components/terminal_event_handler.py +++ b/lib/tfw/components/terminal_event_handler.py @@ -29,9 +29,16 @@ class TerminalEventHandler(EventHandlerBase): self.working_directory = TFWENV.TERMINADO_DIR self._historymonitor = monitor bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash'] - self.terminado_server = TerminadoMiniServer('/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, bash_as_user_cmd) - self.commands = {'write': self.write, - 'read': self.read} + + self.terminado_server = TerminadoMiniServer( + '/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, bash_as_user_cmd + ) + + self.commands = { + 'write': self.write, + 'read': self.read + } + if self._historymonitor: self._historymonitor.watch() self.terminado_server.listen() From d5b0bb4d324a314f72c6f8c57f060fbd4d7b36b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 2 Jun 2018 11:25:48 +0200 Subject: [PATCH 14/18] Avoid potential TypeErrors using LazyInitialise --- lib/tfw/decorators/lazy_initialise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/decorators/lazy_initialise.py b/lib/tfw/decorators/lazy_initialise.py index 615721b..4fdd3a3 100644 --- a/lib/tfw/decorators/lazy_initialise.py +++ b/lib/tfw/decorators/lazy_initialise.py @@ -13,7 +13,7 @@ class LazyInitialise: def __get__(self, instance, owner): if instance is None: - raise TypeError('Cannot get object property from class!') + return self # avoids potential __new__ TypeError value = self.func(instance) setattr(instance, self.func.__name__, value) return value From edc46a8ae67598281c442e6064bde4b79e623189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 2 Jun 2018 11:48:34 +0200 Subject: [PATCH 15/18] Rename LazyInitialise to make IDEs recognise it as a property --- lib/envvars/__init__.py | 4 ++-- lib/tfw/decorators/__init__.py | 2 +- lib/tfw/decorators/{lazy_initialise.py => lazy_property.py} | 2 +- lib/tfw/mixins/callback_mixin.py | 4 ++-- lib/tfw/mixins/observer_mixin.py | 4 ++-- lib/tfw/mixins/supervisor_mixin.py | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) rename lib/tfw/decorators/{lazy_initialise.py => lazy_property.py} (96%) diff --git a/lib/envvars/__init__.py b/lib/envvars/__init__.py index 11faa64..b5d9503 100644 --- a/lib/envvars/__init__.py +++ b/lib/envvars/__init__.py @@ -4,7 +4,7 @@ from collections import namedtuple from os import environ -from tfw.decorators import LazyInitialise +from tfw.decorators import lazy_property class LazyEnvironment: @@ -12,7 +12,7 @@ class LazyEnvironment: self._prefix = prefix self._tuple_name = tuple_name - @LazyInitialise + @lazy_property def environment(self): return self.prefixed_envvars_to_namedtuple() diff --git a/lib/tfw/decorators/__init__.py b/lib/tfw/decorators/__init__.py index cd61318..ed79d7f 100644 --- a/lib/tfw/decorators/__init__.py +++ b/lib/tfw/decorators/__init__.py @@ -2,4 +2,4 @@ # All Rights Reserved. See LICENSE file for details. from .rate_limiter import RateLimiter -from .lazy_initialise import LazyInitialise +from .lazy_property import lazy_property diff --git a/lib/tfw/decorators/lazy_initialise.py b/lib/tfw/decorators/lazy_property.py similarity index 96% rename from lib/tfw/decorators/lazy_initialise.py rename to lib/tfw/decorators/lazy_property.py index 4fdd3a3..80ddbeb 100644 --- a/lib/tfw/decorators/lazy_initialise.py +++ b/lib/tfw/decorators/lazy_property.py @@ -2,7 +2,7 @@ # All Rights Reserved. See LICENSE file for details. -class LazyInitialise: +class lazy_property: """ Decorator that replaces a function with the value it calculates on the first call. diff --git a/lib/tfw/mixins/callback_mixin.py b/lib/tfw/mixins/callback_mixin.py index 971de5c..de0de6c 100644 --- a/lib/tfw/mixins/callback_mixin.py +++ b/lib/tfw/mixins/callback_mixin.py @@ -3,11 +3,11 @@ from functools import partial -from tfw.decorators import LazyInitialise +from tfw.decorators import lazy_property class CallbackMixin: - @LazyInitialise + @lazy_property def _callbacks(self): return [] diff --git a/lib/tfw/mixins/observer_mixin.py b/lib/tfw/mixins/observer_mixin.py index 198c405..712a31e 100644 --- a/lib/tfw/mixins/observer_mixin.py +++ b/lib/tfw/mixins/observer_mixin.py @@ -3,11 +3,11 @@ from watchdog.observers import Observer -from tfw.decorators import LazyInitialise +from tfw.decorators import lazy_property class ObserverMixin: - @LazyInitialise + @lazy_property def observer(self): return Observer() diff --git a/lib/tfw/mixins/supervisor_mixin.py b/lib/tfw/mixins/supervisor_mixin.py index bdd43ea..00cf398 100644 --- a/lib/tfw/mixins/supervisor_mixin.py +++ b/lib/tfw/mixins/supervisor_mixin.py @@ -6,12 +6,12 @@ from xmlrpc.client import Fault as SupervisorFault from contextlib import suppress from os import remove -from tfw.decorators import LazyInitialise +from tfw.decorators import lazy_property from tfw.config import TFWENV class SupervisorBaseMixin: - @LazyInitialise + @lazy_property def supervisor(self): return xmlrpc.client.ServerProxy(TFWENV.SUPERVISOR_HTTP_URI).supervisor From 92e9812776e56cb1cf31127fed39cb4d247a7bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 4 Jun 2018 21:20:36 +0200 Subject: [PATCH 16/18] Remove unnecessary additional_dirs IdeEH.__init__ argument --- lib/tfw/components/ide_event_handler.py | 27 ++++++++++++------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/tfw/components/ide_event_handler.py b/lib/tfw/components/ide_event_handler.py index 4c4fa6f..72fc073 100644 --- a/lib/tfw/components/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler.py @@ -106,17 +106,13 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): The API of each command is documented in their respective handler. """ - def __init__(self, key, directory, allowed_directories, selected_file=None, exclude=None, - additional_watched_directories=None): + def __init__(self, key, directory, allowed_directories, selected_file=None, exclude=None): """ :param key: the key this instance should listen to :param directory: working directory which the EventHandler should serve files from :param allowed_directories: list of directories that can be switched to using the selectdir command :param selected_file: file that is selected by default :param exclude: list of filenames that should not appear between files (for .o, .pyc, etc.) - :param additional_watched_directories: refresh the selected file when files change in these directories - (the working directory is watched by default, this is useful for - symlinks and such) """ super().__init__(key) try: @@ -125,16 +121,19 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): except IndexError: raise EnvironmentError(f'No file(s) in IdeEventHandler working_directory "{directory}"!') - self.watched_directories = [self.filemanager.workdir] - if additional_watched_directories: - self.watched_directories.extend(additional_watched_directories) - MonitorManagerMixin.__init__(self, DirectoryMonitor, self.watched_directories) + MonitorManagerMixin.__init__( + self, + DirectoryMonitor, + self.filemanager.allowed_directories + ) - self.commands = {'read': self.read, - 'write': self.write, - 'select': self.select, - 'selectdir': self.select_dir, - 'exclude': self.exclude} + self.commands = { + 'read': self.read, + 'write': self.write, + 'select': self.select, + 'selectdir': self.select_dir, + 'exclude': self.exclude + } def read(self, data): """ From afc84e1d1a1c2bc71292d0c88ebfa0eb06d04b77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 4 Jun 2018 21:47:10 +0200 Subject: [PATCH 17/18] Conciliate FileManager attribute names and formatting --- lib/tfw/components/ide_event_handler.py | 30 ++++++++++++++++--------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/lib/tfw/components/ide_event_handler.py b/lib/tfw/components/ide_event_handler.py index 72fc073..327f6ef 100644 --- a/lib/tfw/components/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler.py @@ -41,8 +41,8 @@ class FileManager: # pylint: disable=too-many-instance-attributes def workdir(self, directory): if not exists(directory) or not isdir(directory): raise EnvironmentError(f'"{directory}" is not a directory!') - if not self._is_in_whitelisted_dir(directory): - raise EnvironmentError(f'Directory "{directory}" is not in whitelist!') + if not self._is_in_allowed_dir(directory): + raise EnvironmentError(f'Directory "{directory}" is not allowed!') self._workdir = directory @property @@ -65,8 +65,11 @@ class FileManager: # pylint: disable=too-many-instance-attributes @property def files(self): - return [self._relpath(file) for file in glob(join(self._workdir, '**/*'), recursive=True) - if isfile(file) and self._is_in_whitelisted_dir(file) and not self._is_blacklisted(file)] + return [self._relpath(file) + for file in glob(join(self._workdir, '**/*'), recursive=True) + if isfile(file) + and self._is_in_allowed_dir(file) + and not self._is_blacklisted(file)] @property def file_contents(self): @@ -78,8 +81,9 @@ class FileManager: # pylint: disable=too-many-instance-attributes with open(self._filepath(self.filename), 'w', errors='surrogateescape') as ofile: ofile.write(value) - def _is_in_whitelisted_dir(self, path): - return any(realpath(path).startswith(allowed_dir) for allowed_dir in self.allowed_directories) + def _is_in_allowed_dir(self, path): + return any(realpath(path).startswith(allowed_dir) + for allowed_dir in self.allowed_directories) def _is_blacklisted(self, file): return any(fnmatchcase(file, blacklisted) for blacklisted in self.exclude) @@ -110,16 +114,22 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin): """ :param key: the key this instance should listen to :param directory: working directory which the EventHandler should serve files from - :param allowed_directories: list of directories that can be switched to using the selectdir command + :param allowed_directories: list of directories that can be switched to using selectdir :param selected_file: file that is selected by default :param exclude: list of filenames that should not appear between files (for .o, .pyc, etc.) """ super().__init__(key) try: - self.filemanager = FileManager(allowed_directories=allowed_directories, working_directory=directory, - selected_file=selected_file, exclude=exclude) + self.filemanager = FileManager( + allowed_directories=allowed_directories, + working_directory=directory, + selected_file=selected_file, + exclude=exclude + ) except IndexError: - raise EnvironmentError(f'No file(s) in IdeEventHandler working_directory "{directory}"!') + raise EnvironmentError( + f'No file(s) in IdeEventHandler working_directory "{directory}"!' + ) MonitorManagerMixin.__init__( self, From d0667253c2f1bdf64d57fca1ae751a9ff70bc2a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 4 Jun 2018 22:16:44 +0200 Subject: [PATCH 18/18] Harmonize code formatting style across project --- lib/envvars/__init__.py | 8 ++-- lib/tfw/components/directory_monitor.py | 6 ++- .../directory_monitoring_event_handler.py | 10 +++-- lib/tfw/components/history_monitor.py | 17 ++++++--- lib/tfw/components/ide_event_handler.py | 23 +++++++---- lib/tfw/components/log_monitor.py | 10 ++++- .../log_monitoring_event_handler.py | 7 +++- .../process_managing_event_handler.py | 18 ++++++--- lib/tfw/components/terminado_mini_server.py | 23 ++++++----- lib/tfw/components/terminal_commands.py | 8 +++- lib/tfw/components/terminal_event_handler.py | 5 ++- lib/tfw/fsm_base.py | 16 ++++---- lib/tfw/linear_fsm.py | 12 +++++- lib/tfw/mixins/callback_mixin.py | 4 +- lib/tfw/networking/message_sender.py | 6 ++- lib/tfw/networking/serialization.py | 10 ++++- lib/tfw/networking/server/tfw_server.py | 38 +++++++++++++------ 17 files changed, 152 insertions(+), 69 deletions(-) diff --git a/lib/envvars/__init__.py b/lib/envvars/__init__.py index b5d9503..d1ed0d4 100644 --- a/lib/envvars/__init__.py +++ b/lib/envvars/__init__.py @@ -17,7 +17,9 @@ class LazyEnvironment: return self.prefixed_envvars_to_namedtuple() def prefixed_envvars_to_namedtuple(self): - envvars = {envvar.replace(self._prefix, '', 1): environ.get(envvar) - for envvar in environ.keys() - if envvar.startswith(self._prefix)} + envvars = { + envvar.replace(self._prefix, '', 1): environ.get(envvar) + for envvar in environ.keys() + if envvar.startswith(self._prefix) + } return namedtuple(self._tuple_name, envvars)(**envvars) diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index 3ad6732..dc72e9e 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -64,8 +64,10 @@ class IdeReloadWatchdogEventHandler(FileSystemWatchdogEventHandler): self.ignore = self.ignore - 1 return LOG.debug(event) - self.uplink.send({'key': 'ide', - 'data': {'command': 'reload'}}) + self.uplink.send({ + 'key': 'ide', + 'data': {'command': 'reload'} + }) def with_monitor_paused(fun): diff --git a/lib/tfw/components/directory_monitoring_event_handler.py b/lib/tfw/components/directory_monitoring_event_handler.py index 03f58e1..b3beeb2 100644 --- a/lib/tfw/components/directory_monitoring_event_handler.py +++ b/lib/tfw/components/directory_monitoring_event_handler.py @@ -17,10 +17,12 @@ class DirectoryMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): self._directory = directory MonitorManagerMixin.__init__(self, DirectoryMonitor, self._directory) - self.commands = {'pause': self.pause, - 'resume': self.resume, - 'ignore': self.ignore, - 'selectdir': self.selectdir} + self.commands = { + 'pause': self.pause, + 'resume': self.resume, + 'ignore': self.ignore, + 'selectdir': self.selectdir + } @property def directory(self): diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index e775c08..9d259ae 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -39,10 +39,14 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC): self.histfile = histfile self._history = [] self._last_length = len(self._history) - self.observer.schedule(CallbackEventHandler([self.histfile], - self._fetch_history, - self._invoke_callbacks), - dirname(self.histfile)) + self.observer.schedule( + CallbackEventHandler( + [self.histfile], + self._fetch_history, + self._invoke_callbacks + ), + dirname(self.histfile) + ) @property def history(self): @@ -53,7 +57,10 @@ class HistoryMonitor(CallbackMixin, ObserverMixin, ABC): with open(self.histfile, 'r') as ifile: pattern = compileregex(self.command_pattern) data = ifile.read() - self._history = [self.sanitize_command(command) for command in findall(pattern, data)] + self._history = [ + self.sanitize_command(command) + for command in findall(pattern, data) + ] @property @abstractmethod diff --git a/lib/tfw/components/ide_event_handler.py b/lib/tfw/components/ide_event_handler.py index 327f6ef..659ade0 100644 --- a/lib/tfw/components/ide_event_handler.py +++ b/lib/tfw/components/ide_event_handler.py @@ -65,11 +65,13 @@ class FileManager: # pylint: disable=too-many-instance-attributes @property def files(self): - return [self._relpath(file) - for file in glob(join(self._workdir, '**/*'), recursive=True) - if isfile(file) - and self._is_in_allowed_dir(file) - and not self._is_blacklisted(file)] + return [ + self._relpath(file) + for file in glob(join(self._workdir, '**/*'), recursive=True) + if isfile(file) + and self._is_in_allowed_dir(file) + and not self._is_blacklisted(file) + ] @property def file_contents(self): @@ -82,11 +84,16 @@ class FileManager: # pylint: disable=too-many-instance-attributes ofile.write(value) def _is_in_allowed_dir(self, path): - return any(realpath(path).startswith(allowed_dir) - for allowed_dir in self.allowed_directories) + return any( + realpath(path).startswith(allowed_dir) + for allowed_dir in self.allowed_directories + ) def _is_blacklisted(self, file): - return any(fnmatchcase(file, blacklisted) for blacklisted in self.exclude) + return any( + fnmatchcase(file, blacklisted) + for blacklisted in self.exclude + ) def _filepath(self, filename): return join(self._workdir, filename) diff --git a/lib/tfw/components/log_monitor.py b/lib/tfw/components/log_monitor.py index d8d1825..2c6ade8 100644 --- a/lib/tfw/components/log_monitor.py +++ b/lib/tfw/components/log_monitor.py @@ -14,7 +14,10 @@ from tfw.mixins import ObserverMixin, SupervisorLogMixin class LogMonitor(ObserverMixin): def __init__(self, process_name, log_tail=0): self.prevent_log_recursion() - event_handler = SendLogWatchdogEventHandler(process_name, log_tail=log_tail) + event_handler = SendLogWatchdogEventHandler( + process_name, + log_tail=log_tail + ) self.observer.schedule( event_handler, event_handler.path @@ -30,7 +33,10 @@ class SendLogWatchdogEventHandler(PatternMatchingWatchdogEventHandler, Superviso def __init__(self, process_name, log_tail=0): self.process_name = process_name self.procinfo = self.supervisor.getProcessInfo(self.process_name) - super().__init__([self.procinfo['stdout_logfile'], self.procinfo['stderr_logfile']]) + super().__init__([ + self.procinfo['stdout_logfile'], + self.procinfo['stderr_logfile'] + ]) self.uplink = ServerUplinkConnector() self.log_tail = log_tail diff --git a/lib/tfw/components/log_monitoring_event_handler.py b/lib/tfw/components/log_monitoring_event_handler.py index eb9782f..c5dd84c 100644 --- a/lib/tfw/components/log_monitoring_event_handler.py +++ b/lib/tfw/components/log_monitoring_event_handler.py @@ -23,7 +23,12 @@ class LogMonitoringEventHandler(EventHandlerBase, MonitorManagerMixin): super().__init__(key) self.process_name = process_name self.log_tail = log_tail - MonitorManagerMixin.__init__(self, LogMonitor, self.process_name, self.log_tail) + MonitorManagerMixin.__init__( + self, + LogMonitor, + self.process_name, + self.log_tail + ) self.command_handlers = { 'process_name': self.handle_process_name, diff --git a/lib/tfw/components/process_managing_event_handler.py b/lib/tfw/components/process_managing_event_handler.py index 0a3d052..3ccf6b1 100644 --- a/lib/tfw/components/process_managing_event_handler.py +++ b/lib/tfw/components/process_managing_event_handler.py @@ -13,9 +13,11 @@ LOG = logging.getLogger(__name__) class ProcessManager(SupervisorMixin, SupervisorLogMixin): def __init__(self): - self.commands = {'start': self.start_process, - 'stop': self.stop_process, - 'restart': self.restart_process} + self.commands = { + 'start': self.start_process, + 'stop': self.stop_process, + 'restart': self.restart_process + } def __call__(self, command, process_name): return self.commands[command](process_name) @@ -50,8 +52,14 @@ class ProcessManagingEventHandler(EventHandlerBase): except SupervisorFault as fault: message['data']['error'] = fault.faultString finally: - message['data']['stdout'] = self.processmanager.read_stdout(data['process_name'], self.log_tail) - message['data']['stderr'] = self.processmanager.read_stderr(data['process_name'], self.log_tail) + message['data']['stdout'] = self.processmanager.read_stdout( + data['process_name'], + self.log_tail + ) + message['data']['stderr'] = self.processmanager.read_stderr( + data['process_name'], + self.log_tail + ) return message except KeyError: LOG.error('IGNORING MESSAGE: Invalid message received: %s', message) diff --git a/lib/tfw/components/terminado_mini_server.py b/lib/tfw/components/terminado_mini_server.py index 9c79b83..3726097 100644 --- a/lib/tfw/components/terminado_mini_server.py +++ b/lib/tfw/components/terminado_mini_server.py @@ -14,15 +14,15 @@ LOG = logging.getLogger(__name__) class TerminadoMiniServer: def __init__(self, url, port, workdir, shellcmd): self.port = port - self._term_manager = SingleTermManager(shell_command=shellcmd, - term_settings={'cwd': workdir}) - self.application = Application( - [( - url, - TerminadoMiniServer.ResetterTermSocket, - {'term_manager': self._term_manager} - )] + self._term_manager = SingleTermManager( + shell_command=shellcmd, + term_settings={'cwd': workdir} ) + self.application = Application([( + url, + TerminadoMiniServer.ResetterTermSocket, + {'term_manager': self._term_manager} + )]) @property def term_manager(self): @@ -46,5 +46,10 @@ class TerminadoMiniServer: if __name__ == '__main__': LOG.info('Terminado Mini Server listening on %s', TFWENV.TERMINADO_PORT) - TerminadoMiniServer('/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, ['bash']).listen() + TerminadoMiniServer( + '/terminal', + TFWENV.TERMINADO_PORT, + TFWENV.TERMINADO_WD, + ['bash'] + ).listen() IOLoop.instance().start() diff --git a/lib/tfw/components/terminal_commands.py b/lib/tfw/components/terminal_commands.py index ae22975..a47852d 100644 --- a/lib/tfw/components/terminal_commands.py +++ b/lib/tfw/components/terminal_commands.py @@ -36,8 +36,12 @@ class TerminalCommands(ABC): self._setup_bashrc_aliases(bashrc) def _build_command_to_implementation_dict(self): - return {self._parse_command_name(fun): getattr(self, fun) for fun in dir(self) - if callable(getattr(self, fun)) and self._is_command_implementation(fun)} + return { + self._parse_command_name(fun): getattr(self, fun) + for fun in dir(self) + if callable(getattr(self, fun)) + and self._is_command_implementation(fun) + } def _setup_bashrc_aliases(self, bashrc): with open(bashrc, 'a') as ofile: diff --git a/lib/tfw/components/terminal_event_handler.py b/lib/tfw/components/terminal_event_handler.py index 5d1956e..ea135b6 100644 --- a/lib/tfw/components/terminal_event_handler.py +++ b/lib/tfw/components/terminal_event_handler.py @@ -31,7 +31,10 @@ class TerminalEventHandler(EventHandlerBase): bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash'] self.terminado_server = TerminadoMiniServer( - '/terminal', TFWENV.TERMINADO_PORT, TFWENV.TERMINADO_WD, bash_as_user_cmd + '/terminal', + TFWENV.TERMINADO_PORT, + TFWENV.TERMINADO_WD, + bash_as_user_cmd ) self.commands = { diff --git a/lib/tfw/fsm_base.py b/lib/tfw/fsm_base.py index 85ac12f..7296627 100644 --- a/lib/tfw/fsm_base.py +++ b/lib/tfw/fsm_base.py @@ -20,13 +20,15 @@ class FSMBase(CallbackMixin): def __init__(self, initial: str = None, accepted_states: List[str] = None): self.accepted_states = accepted_states or [self.states[-1]] - self.machine = Machine(model=self, - states=self.states, - transitions=self.transitions, - initial=initial or self.states[0], - send_event=True, - ignore_invalid_triggers=True, - after_state_change='execute_callbacks') + self.machine = Machine( + model=self, + states=self.states, + transitions=self.transitions, + initial=initial or self.states[0], + send_event=True, + ignore_invalid_triggers=True, + after_state_change='execute_callbacks' + ) def execute_callbacks(self, event_data): self._execute_callbacks(event_data.kwargs) diff --git a/lib/tfw/linear_fsm.py b/lib/tfw/linear_fsm.py index 61e4c43..1f0e3af 100644 --- a/lib/tfw/linear_fsm.py +++ b/lib/tfw/linear_fsm.py @@ -16,6 +16,14 @@ class LinearFSM(FSMBase): self.states = list(map(str, range(number_of_steps))) self.transitions = [] for index in self.states[:-1]: - self.transitions.append({'trigger': f'step_{int(index)+1}', 'source': index, 'dest': str(int(index)+1)}) - self.transitions.append({'trigger': 'step_next', 'source': index, 'dest': str(int(index)+1)}) + self.transitions.append({ + 'trigger': f'step_{int(index)+1}', + 'source': index, + 'dest': str(int(index)+1) + }) + self.transitions.append({ + 'trigger': 'step_next', + 'source': index, + 'dest': str(int(index)+1) + }) super(LinearFSM, self).__init__() diff --git a/lib/tfw/mixins/callback_mixin.py b/lib/tfw/mixins/callback_mixin.py index de0de6c..3c94e71 100644 --- a/lib/tfw/mixins/callback_mixin.py +++ b/lib/tfw/mixins/callback_mixin.py @@ -15,8 +15,8 @@ class CallbackMixin: """ Subscribe a callable to invoke once an event is triggered. :param callback: callable to be executed on events - :param *args: arguments passed to callable - :param **kwargs: kwargs passed to callable + :param args: arguments passed to callable + :param kwargs: kwargs passed to callable """ fun = partial(callback, *args, **kwargs) self._callbacks.append(fun) diff --git a/lib/tfw/networking/message_sender.py b/lib/tfw/networking/message_sender.py index d1bf0d5..2210d0a 100644 --- a/lib/tfw/networking/message_sender.py +++ b/lib/tfw/networking/message_sender.py @@ -26,5 +26,7 @@ class MessageSender: 'timestamp': datetime.now().isoformat(), 'message': message } - self.server_connector.send({'key': self.key, - 'data': data}) + self.server_connector.send({ + 'key': self.key, + 'data': data + }) diff --git a/lib/tfw/networking/serialization.py b/lib/tfw/networking/serialization.py index 1eaf933..6a6a3e7 100644 --- a/lib/tfw/networking/serialization.py +++ b/lib/tfw/networking/serialization.py @@ -43,11 +43,17 @@ def deserialize_tfw_msg(*args): def _serialize_all(*args): - return tuple(_serialize_single(arg) for arg in args) + return tuple( + _serialize_single(arg) + for arg in args + ) def _deserialize_all(*args): - return tuple(_deserialize_single(arg) for arg in args) + return tuple( + _deserialize_single(arg) + for arg in args + ) def _serialize_single(data): diff --git a/lib/tfw/networking/server/tfw_server.py b/lib/tfw/networking/server/tfw_server.py index 89ad3d4..78cc8ee 100644 --- a/lib/tfw/networking/server/tfw_server.py +++ b/lib/tfw/networking/server/tfw_server.py @@ -30,13 +30,16 @@ class TFWServer: self._fsm.subscribe_callback(self._fsm_updater.update) self._event_handler_connector = EventHandlerConnector() - self.application = Application( - [(r'/ws', ZMQWebSocketProxy, {'make_eventhandler_message': self.make_eventhandler_message, - 'proxy_filter': self.proxy_filter, - 'handle_trigger': self.handle_trigger, - 'event_handler_connector': self._event_handler_connector})] + self.application = Application([( + r'/ws', ZMQWebSocketProxy,{ + 'make_eventhandler_message': self.make_eventhandler_message, + 'proxy_filter': self.proxy_filter, + 'handle_trigger': self.handle_trigger, + 'event_handler_connector': self._event_handler_connector + })] ) - #self.controller_responder = ControllerResponder(self.fsm) TODO: add this once controller stuff is resolved + # self.controller_responder = ControllerResponder(self.fsm) + # TODO: add this once controller stuff is resolved @property def fsm(self): @@ -97,8 +100,11 @@ class FSMManager: self.trigger_predicates[trigger].extend(predicates) def unsubscribe_predicate(self, trigger, *predicates): - self.trigger_predicates[trigger] = [predicate for predicate in self.trigger_predicates[trigger] - not in predicates] + self.trigger_predicates[trigger] = [ + predicate + for predicate in self.trigger_predicates[trigger] + not in predicates + ] class FSMUpdater: @@ -111,10 +117,18 @@ class FSMUpdater: self.uplink.send(self.generate_fsm_update()) def generate_fsm_update(self): - return {'key': 'FSMUpdate', - 'data': self.get_fsm_state_and_transitions()} + return { + 'key': 'FSMUpdate', + 'data': self.get_fsm_state_and_transitions() + } def get_fsm_state_and_transitions(self): state = self.fsm.state - valid_transitions = [{'trigger': trigger} for trigger in self.fsm.machine.get_triggers(self.fsm.state)] - return {'current_state': state, 'valid_transitions': valid_transitions} + valid_transitions = [ + {'trigger': trigger} + for trigger in self.fsm.machine.get_triggers(self.fsm.state) + ] + return { + 'current_state': state, + 'valid_transitions': valid_transitions + }