From 8fab3d22262257a7675d9eac1f93a35bf2b27564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 24 Jul 2018 17:17:14 +0200 Subject: [PATCH 01/68] Conform new fsm_update API --- controller/opt/server.py | 2 +- solvable/src/event_handler_main.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/controller/opt/server.py b/controller/opt/server.py index 82a3811..8155726 100644 --- a/controller/opt/server.py +++ b/controller/opt/server.py @@ -15,7 +15,7 @@ class ControllerPostHandler(RequestHandler): def post(self, *args, **kwargs): self.set_header('Content-Type', 'application/json') self.write(json.dumps({ - 'solved': self.controller.in_accepted_state + 'solved': self.controller.fsm_in_accepted_state })) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 9ff99e5..36a8902 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -86,14 +86,15 @@ class MessageFSMStepsEventHandler(FSMAwareEventHandler): def handle_event(self, message): pass - def handle_fsm_step(self, from_state, to_state, trigger): + def handle_fsm_step(self, **kwargs): """ When the FSM steps this method is invoked. + Receives a 'data' field from an fsm_update message as kwargs. """ MessageSender().send( 'FSM info', - f'FSM has stepped from state "{from_state}" ' - f'to state "{to_state}" in response to trigger "{trigger}"' + f'FSM has stepped from state "{kwargs["last_event"]["from_state"]}" ' + f'to state "{kwargs["current_state"]}" in response to trigger "{kwargs["last_event"]["trigger"]}"' ) From 265560b6144c4dc8280a3d5e4815dcaea279e14f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 26 Jul 2018 14:00:24 +0200 Subject: [PATCH 02/68] Conform new import routes --- controller/Dockerfile | 1 - controller/opt/server.py | 2 +- solvable/src/event_handler_main.py | 3 ++- solvable/src/test_fsm.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/controller/Dockerfile b/controller/Dockerfile index ef96d41..d09b2fd 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -7,7 +7,6 @@ ENV PYTHONPATH="/usr/local/lib" \ TFW_AUTH_KEY="/tmp/tfw-auth.key" \ CONTROLLER_PORT=5555 -RUN pip3 install watchdog transitions COPY ./controller/ / CMD ["python3", "/opt/server.py"] diff --git a/controller/opt/server.py b/controller/opt/server.py index 8155726..175d738 100644 --- a/controller/opt/server.py +++ b/controller/opt/server.py @@ -4,7 +4,7 @@ import json from tornado.ioloop import IOLoop from tornado.web import RequestHandler, Application -from tfw import FSMAwareEventHandler +from tfw.event_handler_base import FSMAwareEventHandler class ControllerPostHandler(RequestHandler): diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 36a8902..1d09f7a 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -3,7 +3,8 @@ from functools import partial from tornado.ioloop import IOLoop -from tfw import YamlFSM, FSMAwareEventHandler +from tfw.fsm import YamlFSM +from tfw.event_handler_base import FSMAwareEventHandler from tfw.components import IdeEventHandler, TerminalEventHandler from tfw.components import ProcessManagingEventHandler, BashMonitor from tfw.components import TerminalCommands, LogMonitoringEventHandler diff --git a/solvable/src/test_fsm.py b/solvable/src/test_fsm.py index 3dcf7c3..2db9b97 100644 --- a/solvable/src/test_fsm.py +++ b/solvable/src/test_fsm.py @@ -2,7 +2,7 @@ from os.path import exists -from tfw import LinearFSM +from tfw.fsm import LinearFSM from tfw.networking import MessageSender From 8b47b301e4d646f881c00cb64fe5f65e705f1fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 27 Jul 2018 13:59:14 +0200 Subject: [PATCH 03/68] Use jinja2 in default YamlFSM --- solvable/src/event_handler_main.py | 6 +++++- solvable/src/test_fsm.yml | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 1d09f7a..a3639e2 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -103,7 +103,11 @@ if __name__ == '__main__': # TFW component EventHandlers (builtins, required for their respective functionalities) fsm = FSMManagingEventHandler( # TFW FSM key='fsm', - fsm_type=partial(YamlFSM, 'test_fsm.yml') + fsm_type=partial( + YamlFSM, + 'test_fsm.yml', + {} # jinja2 variables, use empty dict to enable jinja2 parsing without any variables + ) ) ide = IdeEventHandler( # Web IDE backend key='ide', diff --git a/solvable/src/test_fsm.yml b/solvable/src/test_fsm.yml index b195479..d34ebcb 100644 --- a/solvable/src/test_fsm.yml +++ b/solvable/src/test_fsm.yml @@ -41,3 +41,8 @@ transitions: - trigger: step_5 source: '4' dest: '5' + {% for i in range(5) %} # you can also use jinja2 in this config file + - trigger: 'step_next' + source: '{{i}}' + dest: '{{i+1}}' + {% endfor %} From db551e87883d848e1b18f8d2d3a87d6ee41bc7ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 27 Jul 2018 13:59:34 +0200 Subject: [PATCH 04/68] Add notice on YamlFSM jinja2 support --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9a8e647..16acabf 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ A good state machine is the backbone of a good TFW challenge. There are two ways to define a state machine: - Using a YAML configuration file - - Implementing it in Python by hand + - Implementing it in Python The first option allows you to handle FSM callbacks and custom logic in any programming language (not just Python) and is generally really easy to work with (you can execute arbitrary shell commands on events). You should choose this method unless you have good reason not to. @@ -230,6 +230,8 @@ It is also possible to add preconditions to transitions. This is done by adding a `predicates` key with a list of shell commands to run. If you do this, the transition will only succeed if the return code of all predicates was `0` (as per unix convention for success). +Our `YamlFSM` implementation also supports jinja2 templates inside the `YAML` config file (examples in `test_fsm.yml`). + ## Baby steps When creating your own challenge the process should be the following: From 5e390e1449a97d590e948bb04b9efa8ce2185902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 30 Jul 2018 08:56:40 +0200 Subject: [PATCH 05/68] Update lies in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 16acabf..b9e1e52 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ Refer to the example in this repo. ### src -This folder contains the source code of a server running TFW and an other server running our event handlers. +This folder contains the source code of our pre-written event handlers and example FSMs. Note that this is not a part of the framework by any means, these are just simple examples. ``` From 38963e7b81d80b1e60fff62173c6c2de4297ea11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 1 Aug 2018 15:05:08 +0200 Subject: [PATCH 06/68] Improve Dockerfile --- solvable/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/solvable/Dockerfile b/solvable/Dockerfile index 7dd51b0..aa6382f 100644 --- a/solvable/Dockerfile +++ b/solvable/Dockerfile @@ -7,9 +7,9 @@ RUN pip3 install Flask==1.0 \ git+https://github.com/avatao-content/tfwconnector.git#subdirectory=python3 # Define variables to use later -ENV TFW_EHMAIN_DIR="/srv/.tfw_builtin_ehs" \ - TFW_WEBSERVICE_DIR="/srv/webservice" \ - TFW_IDE_WD="/home/${AVATAO_USER}/workdir" \ +ENV TFW_EHMAIN_DIR="${TFW_DIR}/builtin_event_handlers" \ + TFW_WEBSERVICE_DIR="/srv/webservice" \ + TFW_IDE_WD="/home/${AVATAO_USER}/workdir" \ TFW_TERMINADO_WD="/home/${AVATAO_USER}/workdir" # Copy TFW related stuff to a dedicated directory @@ -26,7 +26,8 @@ RUN mkdir -p ${TFW_IDE_WD} &&\ chmod -R 755 "${TFW_IDE_WD}" "${TFW_WEBSERVICE_DIR}" # Hide TFW related code from user -RUN chown -R root:root ${TFW_SERVER_DIR} && chmod -R 700 ${TFW_SERVER_DIR} +RUN chown -R root:root ${TFW_SERVER_DIR} ${TFW_DIR} &&\ + chmod -R 700 ${TFW_SERVER_DIR} ${TFW_DIR} # Make AVATAO_USER's home writeable and set it as WORKDIR # Make webservice directory writable From 829c2448675c02bb5eb86183176aa388460c9b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 3 Aug 2018 13:33:11 +0200 Subject: [PATCH 07/68] Fix HOTRELOAD --- hack/libhack/challenge.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/libhack/challenge.sh b/hack/libhack/challenge.sh index 8a7e558..4ed1377 100644 --- a/hack/libhack/challenge.sh +++ b/hack/libhack/challenge.sh @@ -40,7 +40,7 @@ challenge::run() { if [[ -d "${BASEIMAGE_PATH}" ]]; then mount_baseimage="-v ${BASEIMAGE_PATH}/lib/tfw:/usr/local/lib/tfw" fi - mount_challenge="-v ${CHALLENGE_PATH}/solvable/src:/srv/.tfw_builtin_ehs" + mount_challenge="-v ${CHALLENGE_PATH}/solvable/src:/.tfw/builtin_event_handlers" mount_volumes="${mount_baseimage:-} ${mount_challenge}" fi popd From 3bda7bc5403e0b7211ce8abc5cf08df9ea4ba957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 7 Aug 2018 17:01:58 +0200 Subject: [PATCH 08/68] Make cleanup() calls automatic in event_handler_main.py --- solvable/src/event_handler_main.py | 34 ++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index a3639e2..4b0b564 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -1,10 +1,12 @@ from ast import literal_eval from functools import partial +from signal import signal, SIGTERM, SIGINT, SIGHUP +from sys import exit as sysexit from tornado.ioloop import IOLoop from tfw.fsm import YamlFSM -from tfw.event_handler_base import FSMAwareEventHandler +from tfw.event_handler_base import EventHandlerBase, FSMAwareEventHandler from tfw.components import IdeEventHandler, TerminalEventHandler from tfw.components import ProcessManagingEventHandler, BashMonitor from tfw.components import TerminalCommands, LogMonitoringEventHandler @@ -99,9 +101,11 @@ class MessageFSMStepsEventHandler(FSMAwareEventHandler): ) -if __name__ == '__main__': +def main(): + # pylint: disable=possibly-unused-variable + # # TFW component EventHandlers (builtins, required for their respective functionalities) - fsm = FSMManagingEventHandler( # TFW FSM + fsm = FSMManagingEventHandler( # TFW FSM key='fsm', fsm_type=partial( YamlFSM, @@ -131,20 +135,32 @@ if __name__ == '__main__': ) # Your custom event handlers - message_fsm_steps = MessageFSMStepsEventHandler( + message_fsm_steps_eh = MessageFSMStepsEventHandler( key='test' ) # Terminal command handlers commands = TestCommands(bashrc=f'/home/{TAOENV.USER}/.bashrc') terminal.historymonitor.subscribe_callback(commands.callback) - - # Example terminal command callback terminal.historymonitor.subscribe_callback(cenator) + # Raise SystemExit on signals we should handle gracefully (make finally run) + trigger_finally = lambda a, b: sysexit() + signal(SIGTERM, trigger_finally) + signal(SIGINT, trigger_finally) + signal(SIGHUP, trigger_finally) + + # Start event handlers and clean up on stopping try: IOLoop.instance().start() finally: - eventhandlers = {fsm, ide, terminal, processmanager, logmonitor, message_fsm_steps} - for eh in eventhandlers: - eh.cleanup() + event_handlers = ( + eh for eh in locals().values() + if isinstance(eh, EventHandlerBase) + ) + for event_handler in event_handlers: + event_handler.cleanup() + + +if __name__ == '__main__': + main() From e7d78ed289492bd9745d52826a40157cb6c848c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 7 Aug 2018 17:03:41 +0200 Subject: [PATCH 09/68] Add snapshot event handler --- solvable/src/event_handler_main.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 4b0b564..b1702d3 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -10,7 +10,7 @@ from tfw.event_handler_base import EventHandlerBase, FSMAwareEventHandler from tfw.components import IdeEventHandler, TerminalEventHandler from tfw.components import ProcessManagingEventHandler, BashMonitor from tfw.components import TerminalCommands, LogMonitoringEventHandler -from tfw.components import FSMManagingEventHandler +from tfw.components import FSMManagingEventHandler, DirectorySnapshottingEventHandler from tfw.networking import MessageSender, TFWServerConnector from tfw.config import TFWENV from tfw.config.logs import logging @@ -133,6 +133,13 @@ def main(): process_name='webservice', log_tail=2000 ) + snapshot = DirectorySnapshottingEventHandler( # Manages filesystem snapshots of directories + key='snapshot', + directories=[ + TFWENV.IDE_WD, + TFWENV.WEBSERVICE_DIR + ] + ) # Your custom event handlers message_fsm_steps_eh = MessageFSMStepsEventHandler( From 4a1073e5246013192f19365e7bf5466bf610581c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 31 Aug 2018 16:57:00 +0200 Subject: [PATCH 10/68] Bootify webservice db session handling --- solvable/src/webservice/model.py | 35 ++++++++++++++++++++----------- solvable/src/webservice/server.py | 24 ++++++++++----------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/solvable/src/webservice/model.py b/solvable/src/webservice/model.py index d722cdc..b084b1e 100644 --- a/solvable/src/webservice/model.py +++ b/solvable/src/webservice/model.py @@ -13,18 +13,29 @@ session_factory = sessionmaker( ) -@contextmanager -def Session(factory=session_factory): - session = factory() - try: - yield session - session.commit() - except: - session.rollback() - raise - # session is closed by flask - # finally: - # session.close() +class SessionWrapper: + def __init__(self): + self._session_factory = session_factory + self._session_handle = None + + @contextmanager + def session(self): + try: + yield self._session + self._session.commit() + except: + self._session.rollback() + raise + + @property + def _session(self): + if self._session_handle is None: + self._session_handle = self._session_factory() + return self._session_handle + + def teardown(self): + if self._session_handle is not None: + self._session_handle.close() Base = declarative_base() diff --git a/solvable/src/webservice/server.py b/solvable/src/webservice/server.py index c359c61..1b36762 100644 --- a/solvable/src/webservice/server.py +++ b/solvable/src/webservice/server.py @@ -1,9 +1,8 @@ from os import urandom, getenv -from functools import partial from flask import Flask, render_template, request, session, url_for, g -from model import init_db, session_factory, Session +from model import init_db, SessionWrapper from user_ops import UserOps from errors import InvalidCredentialsError, UserExistsError @@ -16,25 +15,24 @@ app.jinja_env.globals.update( # pylint: disable=no-member ) -def get_db_session(): - if not hasattr(g, 'db_session'): - g.db_session = session_factory() - return g.db_session - -Session = partial(Session, get_db_session) +@app.before_request +def setup_db(): + # pylint: disable=protected-access + g._db_session_wrapper = SessionWrapper() + g.db_session = g._db_session_wrapper.session @app.teardown_appcontext -def close_db_session(err): # pylint: disable=unused-argument - if hasattr(g, 'db_session'): - g.db_session.close() +def close_db_session(_): + # pylint: disable=protected-access + g._db_session_wrapper.teardown() @app.route('/', methods=['GET', 'POST']) def index(): if request.method == 'POST': try: - with Session() as db_session: + with g.db_session() as db_session: UserOps( request.form.get('username'), request.form.get('password'), @@ -67,7 +65,7 @@ def register(): return render_template('register.html', alert='Passwords do not match! Please try again.') try: - with Session() as db_session: + with g.db_session() as db_session: UserOps( request.form.get('username'), request.form.get('password'), From 97c593640d41208399cb1fd02e720d9022d27544 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 17 Sep 2018 11:49:10 +0200 Subject: [PATCH 11/68] Add macOS grep aliases to hack scripts --- hack/bootstrap.sh | 2 +- hack/tfw.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/bootstrap.sh b/hack/bootstrap.sh index 643a9cc..a6aedfa 100755 --- a/hack/bootstrap.sh +++ b/hack/bootstrap.sh @@ -3,7 +3,7 @@ set -eu set -o pipefail set -o errtrace shopt -s expand_aliases -[ "$(uname)" == "Darwin" ] && alias sed="gsed" || : +[ "$(uname)" == "Darwin" ] && alias sed="gsed" && alias grep="ggrep" || : HERE="$(pwd)" CHALLENGE=${CHALLENGE:-test-tutorial-framework} diff --git a/hack/tfw.sh b/hack/tfw.sh index 46a843b..f05f2b7 100755 --- a/hack/tfw.sh +++ b/hack/tfw.sh @@ -3,7 +3,7 @@ set -eu set -o pipefail set -o errtrace shopt -s expand_aliases -[ "$(uname)" == "Darwin" ] && alias readlink="greadlink" || : +[ "$(uname)" == "Darwin" ] && alias readlink="greadlink" && alias grep="ggrep" || : SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" TFW_PATH="${TFW_PATH:-$SCRIPT_DIR/../..}" From 89c55651269edfc12cdc14b2535d70b7e4b7604b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 13 Feb 2019 13:51:55 +0100 Subject: [PATCH 12/68] Add missing GNU utils to bootstrap script dep list --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index b9e1e52..4b69354 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,9 @@ Dependencies: - yarn - Angular CLI - GNU coreutils +- GNU findutils +- GNU sed +- GNU grep Just copy and paste the following command in a terminal: From a1b4a878641dd096e2b047e377d3f23045e481f6 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 20 May 2019 11:09:27 +0200 Subject: [PATCH 13/68] Replace cleanup() with stop() --- solvable/src/event_handler_main.py | 8 ++++---- solvable/src/pipe_io_main.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 5be6423..39e38fd 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -151,12 +151,12 @@ def main(): terminal.historymonitor.subscribe_callback(cenator) event_handlers = EventHandlerBase.get_local_instances() - def cleanup(sig, frame): + def stop(sig, frame): for eh in event_handlers: - eh.cleanup() + eh.stop() exit(0) - signal(SIGTERM, cleanup) - signal(SIGINT, cleanup) + signal(SIGTERM, stop) + signal(SIGINT, stop) IOLoop.instance().start() diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index 1edd1ab..c81787e 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -14,11 +14,11 @@ if __name__ == '__main__': ) event_handlers = EventHandlerBase.get_local_instances() - def cleanup(sig, frame): + def stop(sig, frame): for eh in event_handlers: - eh.cleanup() + eh.stop() exit(0) - signal(SIGTERM, cleanup) - signal(SIGINT, cleanup) + signal(SIGTERM, stop) + signal(SIGINT, stop) IOLoop.instance().start() From 48efb39ea58c5efcb78500960fe38f26bf14ff60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sun, 26 May 2019 18:31:16 +0200 Subject: [PATCH 14/68] Partially conform new TFW networking model --- solvable/src/event_handler_main.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 39e38fd..71652ab 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -10,6 +10,7 @@ from tfw.components import IdeEventHandler, TerminalEventHandler from tfw.components import ProcessManagingEventHandler, BashMonitor from tfw.components import TerminalCommands, LogMonitoringEventHandler from tfw.components import FSMManagingEventHandler, DirectorySnapshottingEventHandler +from tfw.components import FrontendEventHandler from tfw.networking import MessageSender, TFWServerConnector from tfw.config import TFWENV from tfw.config.logs import logging @@ -44,7 +45,7 @@ class TestCommands(TerminalCommands): """ if not args: message_template = """'{"key": "", "data": {"command": ""}}'""" - TFWServerConnector().send_to_eventhandler({ + TFWServerConnector().send_message({ 'key': 'shell', 'data': { 'command': 'write', @@ -52,7 +53,7 @@ class TestCommands(TerminalCommands): } }) else: - TFWServerConnector().send(literal_eval(args[0])) + TFWServerConnector().send_message(literal_eval(args[0])) def command_seppuku_tfw(self, *args): """ @@ -65,14 +66,14 @@ class TestCommands(TerminalCommands): 'clear && echo "Committed seppuku! :)" && sleep infinity' ) uplink = TFWServerConnector() - uplink.send_to_eventhandler({ + uplink.send_message({ 'key': 'shell', 'data': { 'command': 'write', 'value': f'{seppuku}\n' } }) - uplink.send({ + uplink.send_message({ 'key': 'dashboard', 'data': { 'command': 'reloadFrontend' @@ -139,6 +140,7 @@ def main(): TFWENV.WEBSERVICE_DIR ] ) + frontend = FrontendEventHandler() # Proxies frontend API calls to frontend # Your custom event handlers message_fsm_steps_eh = MessageFSMStepsEventHandler( From 2796b26d92dcfd8df9c81aa9df8928c2e044d9d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 4 Jun 2019 13:59:12 +0200 Subject: [PATCH 15/68] Harmonize pylintrc with baseimage repo --- .pylintrc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 7fb46c0..1cbae5d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,5 +1,12 @@ [TYPECHECK] ignored-modules = zmq -max-line-length = 150 +max-line-length = 120 disable = missing-docstring, too-few-public-methods, invalid-name + +[SIMILARITIES] + +min-similarity-lines=6 +ignore-comments=yes +ignore-docstrings=yes +ignore-imports=yes From 0104a209bf550e8e19fc9f6142ff759067026b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 4 Jun 2019 13:59:32 +0200 Subject: [PATCH 16/68] Comply TFW lib changes --- controller/opt/server.py | 2 +- solvable/src/event_handler_main.py | 11 +++++------ solvable/src/pipe_io_main.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/controller/opt/server.py b/controller/opt/server.py index 175d738..9e211c5 100644 --- a/controller/opt/server.py +++ b/controller/opt/server.py @@ -4,7 +4,7 @@ import json from tornado.ioloop import IOLoop from tornado.web import RequestHandler, Application -from tfw.event_handler_base import FSMAwareEventHandler +from tfw.event_handlers import FSMAwareEventHandler class ControllerPostHandler(RequestHandler): diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 71652ab..1e9ea96 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -5,13 +5,12 @@ from signal import signal, SIGTERM, SIGINT from tornado.ioloop import IOLoop from tfw.fsm import YamlFSM -from tfw.event_handler_base import EventHandlerBase, FSMAwareEventHandler +from tfw.event_handlers import EventHandlerBase, FSMAwareEventHandler, TFWServerUplinkConnector from tfw.components import IdeEventHandler, TerminalEventHandler from tfw.components import ProcessManagingEventHandler, BashMonitor from tfw.components import TerminalCommands, LogMonitoringEventHandler from tfw.components import FSMManagingEventHandler, DirectorySnapshottingEventHandler -from tfw.components import FrontendEventHandler -from tfw.networking import MessageSender, TFWServerConnector +from tfw.components import FrontendEventHandler, MessageSender from tfw.config import TFWENV from tfw.config.logs import logging from tao.config import TAOENV @@ -45,7 +44,7 @@ class TestCommands(TerminalCommands): """ if not args: message_template = """'{"key": "", "data": {"command": ""}}'""" - TFWServerConnector().send_message({ + TFWServerUplinkConnector().send_message({ 'key': 'shell', 'data': { 'command': 'write', @@ -53,7 +52,7 @@ class TestCommands(TerminalCommands): } }) else: - TFWServerConnector().send_message(literal_eval(args[0])) + TFWServerUplinkConnector().send_message(literal_eval(args[0])) def command_seppuku_tfw(self, *args): """ @@ -65,7 +64,7 @@ class TestCommands(TerminalCommands): 'nohup sh -c "supervisorctl restart tfwserver event_handler_main" &> /dev/null & ' 'clear && echo "Committed seppuku! :)" && sleep infinity' ) - uplink = TFWServerConnector() + uplink = TFWServerUplinkConnector() uplink.send_message({ 'key': 'shell', 'data': { diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index c81787e..d750d5b 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -2,7 +2,7 @@ from signal import signal, SIGTERM, SIGINT from tornado.ioloop import IOLoop -from tfw.event_handler_base import EventHandlerBase +from tfw.event_handlers import EventHandlerBase from tfw.components import PipeIOEventHandler From 8b04bc047791f87da7e72f249cbf89dc9f75ced6 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 10 Jun 2019 15:35:09 +0200 Subject: [PATCH 17/68] Import the logging module directly --- solvable/src/event_handler_main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 1e9ea96..5654713 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -1,3 +1,4 @@ +import logging from ast import literal_eval from functools import partial from signal import signal, SIGTERM, SIGINT @@ -12,10 +13,10 @@ from tfw.components import TerminalCommands, LogMonitoringEventHandler from tfw.components import FSMManagingEventHandler, DirectorySnapshottingEventHandler from tfw.components import FrontendEventHandler, MessageSender from tfw.config import TFWENV -from tfw.config.logs import logging from tao.config import TAOENV LOG = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) def cenator(history): From 6b14cd0caeed5a5acdfc4002feb3ed5dbe8d7925 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 11 Jun 2019 11:32:46 +0200 Subject: [PATCH 18/68] Configure loggging in pipe_io_main(), too --- solvable/src/pipe_io_main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index d750d5b..ca6bb45 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -1,3 +1,4 @@ +import logging from signal import signal, SIGTERM, SIGINT from tornado.ioloop import IOLoop @@ -5,6 +6,8 @@ from tornado.ioloop import IOLoop from tfw.event_handlers import EventHandlerBase from tfw.components import PipeIOEventHandler +logging.basicConfig(level=logging.DEBUG) + if __name__ == '__main__': pipe_io = PipeIOEventHandler( From 14f300d6103ae9f7440282278e10c98d0ecff7d4 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 4 Jun 2019 15:25:55 +0200 Subject: [PATCH 19/68] Include PipeIO dependencies --- solvable/src/pipe_io_auxlib.py | 235 +++++++++++++++++++++++++++++++++ solvable/src/pipe_io_main.py | 59 ++++++++- 2 files changed, 291 insertions(+), 3 deletions(-) create mode 100644 solvable/src/pipe_io_auxlib.py diff --git a/solvable/src/pipe_io_auxlib.py b/solvable/src/pipe_io_auxlib.py new file mode 100644 index 0000000..a06ce06 --- /dev/null +++ b/solvable/src/pipe_io_auxlib.py @@ -0,0 +1,235 @@ +from json import dumps, loads + +from tfw.crypto import KeyManager, sign_message, verify_message +from tfw.components import PipeIOEventHandlerBase +from tfw.components.pipe_io_event_handler import DEFAULT_PERMISSIONS + + +class SignMessagePipeIOEventHandler(PipeIOEventHandlerBase): + """ + Signs a valid TFW message with HMAC. + Note that the running process needs root permissions in order to read + the authentication key. + When forwarding is true, it will send the signed message to the TFW + server before writing it into the output pipe. + """ + + def __init__( + self, in_pipe_path, out_pipe_path, + permissions=DEFAULT_PERMISSIONS, + forwarding=True + ): + self.forwarding = forwarding + self.auth_key = KeyManager().auth_key + super().__init__(None, in_pipe_path, out_pipe_path, permissions) + + def handle_event(self, message): + pass + + def handle_pipe_event(self, message_bytes): + message = loads(message_bytes) + sign_message(self.auth_key, message) + if self.forwarding: + self.server_connector.send(message) + self.pipe_io.send_message(dumps(message).encode()) + + +class VerifyMessagePipeIOEventHandler(PipeIOEventHandlerBase): + """ + Verifies a signed TFW message. + This pipe also needs root permissions. Send the serialized JSON object + to the pipe, then wait for its boolean response. + """ + + def __init__(self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS): + self.auth_key = KeyManager().auth_key + super().__init__(None, in_pipe_path, out_pipe_path, permissions) + + def handle_event(self, message): + pass + + def handle_pipe_event(self, message_bytes): + message = loads(message_bytes) + validity = verify_message(self.auth_key, message) + self.pipe_io.send_message(str(validity).lower().encode()) + + +class BotPipeIOEventHandler(PipeIOEventHandlerBase): + """ + Sends bot messages to the frontend. + If you assign @originator, it will be the default message sender. + When you write a line to the pipe, it will be considered as a single + message and gets appended to the queue until an empty line is received, + which triggers forwarding the messages to the TFW server. + """ + + def __init__( + self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS, + originator='avataobot' + ): + self.queue = [] + self.originator = originator + super().__init__(None, in_pipe_path, out_pipe_path, permissions) + + def handle_event(self, message): + pass + + def handle_pipe_event(self, message_bytes): + if message_bytes == b"": + if self.queue: + self.server_connector.send({ + 'key': 'queueMessages', + 'data': { + 'messages': self.queue + } + }) + self.queue = [] + else: + self.queue.append({ + 'originator': self.originator, + 'message': message_bytes.decode().replace('\\n', '\n') + }) + + +class DeployPipeIOEventHandler(PipeIOEventHandlerBase): + """ + Manages deployment in the IDE. + When you receive "deploy", then you have to answer with a "true" or + "false" depending whether you are satisfied with the result or not. + The @process parameter is the name of the supervised service. + """ + + # pylint: disable=too-many-arguments + def __init__( + self, in_pipe_path, out_pipe_path, process, + permissions=DEFAULT_PERMISSIONS + ): + self.expected = False + self.process = process + + self.onsuccess = { + 'key': 'processmanager', + 'data': { + 'process_name': process, + 'command': 'restart' + } + } + self.onerror = { + 'key': 'processmanager', + 'data': { + 'process_name': process, + 'error': True + } + } + + super().__init__('processmanager', in_pipe_path, out_pipe_path, permissions) + + def handle_event(self, message): + if message == self.onsuccess: + self.expected = True + self.pipe_io.send_message(b'deploy') + + def handle_pipe_event(self, message_bytes): + if not self.expected: + raise ValueError( + f'{self.pipe_io.in_pipe}: There is nothing to deploy.' + ) + + self.expected = False + if message_bytes == b'true': + self.server_connector.send(self.onsuccess) + elif message_bytes == b'false': + self.server_connector.send(self.onerror) + else: + raise ValueError( + f'{self.pipe_io.in_pipe}: Expected "true" or "false".' + ) + + +class IdePipeIOEventHandler(PipeIOEventHandlerBase): + """ + Manipulates the content of the IDE. + You can observe a file, and when the user edits it, you will receive + the new contents where newlines are escaped as "\\n". + In order to overwrite the file, send an escaped text back to the pipe. + Since the pipe doesn't know if the file is selected initially in the IDE, + you have to provide this information by yourself with @selected, + but it will track it later on. + """ + + # pylint: disable=too-many-arguments + def __init__( + self, in_pipe_path, out_pipe_path, filename, + permissions=DEFAULT_PERMISSIONS, + selected=True + ): + self.selected = selected + self.filename = filename + super().__init__('ide', in_pipe_path, out_pipe_path, permissions) + + def handle_event(self, message): + data = message['data'] + + if data['command'] == 'select': + self.selected = data['filename'] == self.filename + elif data['command'] == 'write' and self.selected: + clean = data['content'].replace('\n', '\\n') + self.pipe_io.send_message(clean.encode()) + + def handle_pipe_event(self, message_bytes): + if not self.selected: + self.server_connector.send({ + 'key': 'mirror', + 'data': { + 'key': 'ide', + 'data': { + 'command': 'select', + 'filename': self.filename + } + } + }) + + self.server_connector.send({ + 'key': 'mirror', + 'data': { + 'key': 'ide', + 'data': { + 'command': 'write', + 'content': message_bytes.decode().replace('\\n', '\n') + } + } + }) + self.server_connector.send({ + 'key': 'mirror', + 'data': { + 'key': 'ide', + 'data': { + 'command': 'read' + } + } + }) + + +class FSMPipeIOEventHandler(PipeIOEventHandlerBase): + """ + Handles FSM steps. + When the FSM enters the next state, you will receive a line containing + its name. To trigger a state change, send the name of the transition to + the pipe. + """ + + def __init__(self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS): + super().__init__( + ['fsm', 'fsm_update'], + in_pipe_path, out_pipe_path, permissions + ) + + def handle_event(self, message): + if 'current_state' in message['data']: + self.pipe_io.send_message(message['data']['current_state'].encode()) + + def handle_pipe_event(self, message_bytes): + self.server_connector.send({ + 'key': '', + 'trigger': message_bytes.decode() + }) diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index ca6bb45..b15f6a5 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -6,14 +6,67 @@ from tornado.ioloop import IOLoop from tfw.event_handlers import EventHandlerBase from tfw.components import PipeIOEventHandler +from pipe_io_auxlib import ( + SignMessagePipeIOEventHandler, VerifyMessagePipeIOEventHandler, + BotPipeIOEventHandler, + DeployPipeIOEventHandler, IdePipeIOEventHandler, + FSMPipeIOEventHandler +) + logging.basicConfig(level=logging.DEBUG) if __name__ == '__main__': - pipe_io = PipeIOEventHandler( + """ + Creates general purpose pipes. + The first parameter associates the receiving pipe with a key, which is + an empty string in this case. It has a special meaning, you can + subscribe to every kind of message with this key. + If you wish to filter incoming data, specify a single or more keys in + a list, eg.: processmanager, ide, key... + You can send/receive JSON messages to/from the TFW server as any user, + because we gave read+write permissions, without that parameter, only + the owner has access to the pipes. + """ + json_pipe = PipeIOEventHandler( '', - '/tmp/tfw_send', - '/tmp/tfw_recv' + '/tmp/tfw_json_send', + '/tmp/tfw_json_recv', + permissions=0o666 + ) + + sign_pipe = SignMessagePipeIOEventHandler( + '/tmp/tfw_sign_send', + '/tmp/tfw_sign_recv', + forwarding=True + ) + + verify_pipe = VerifyMessagePipeIOEventHandler( + '/tmp/tfw_verify_send', + '/tmp/tfw_verify_recv' + ) + + bot_pipe = BotPipeIOEventHandler( + '/tmp/tfw_bot_send', + '/tmp/tfw_bot_recv' + ) + + deploy_pipe = DeployPipeIOEventHandler( + '/tmp/tfw_deploy_send', + '/tmp/tfw_deploy_recv', + 'webservice' + ) + + ide_pipe = IdePipeIOEventHandler( + '/tmp/tfw_ide_send', + '/tmp/tfw_ide_recv', + 'user_ops.py', + selected=True + ) + + fsm_pipe = FSMPipeIOEventHandler( + '/tmp/tfw_fsm_send', + '/tmp/tfw_fsm_recv' ) event_handlers = EventHandlerBase.get_local_instances() From 8b15da39ae8a8937caaf693e84e31f3e76b77abc Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 4 Jun 2019 15:44:47 +0200 Subject: [PATCH 20/68] Exterminate TFWConnector from TestFSM --- solvable/src/test_fsm.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/solvable/src/test_fsm.yml b/solvable/src/test_fsm.yml index d34ebcb..793048d 100644 --- a/solvable/src/test_fsm.yml +++ b/solvable/src/test_fsm.yml @@ -4,25 +4,21 @@ states: - name: '0' - name: '1' on_enter: | - python3 -c "from tfwconnector import MessageSender; MessageSender().send('FSM', 'Entered state 1!')" + printf "Entered state 1!\n\n" > /tmp/tfw_bot_send - name: '2' on_enter: | file=/home/user/workdir/cat.txt echo "As you can see it is possible to execute arbitrary shell commands here." >> $file - python3 -c \ - " - from tfwconnector import MessageSender - MessageSender().send('FSM', 'Entered state 2! Written stuff to $file') - " + printf "Entered state 2! Written stuff to $file\n\n" > /tmp/tfw_bot_send - name: '3' on_enter: | - python3 -c "from tfwconnector import MessageSender; MessageSender().send('FSM', 'Entered state 3!')" + printf "Entered state 3!\n\n" > /tmp/tfw_bot_send - name: '4' on_enter: | - python3 -c "from tfwconnector import MessageSender; MessageSender().send('FSM', 'Entered state 4!')" + printf "Entered state 4!\n\n" > /tmp/tfw_bot_send - name: '5' on_enter: | - python3 -c "from tfwconnector import MessageSender; MessageSender().send('FSM', 'Entered state 5!')" + printf "Entered state 5!\n\n" > /tmp/tfw_bot_send transitions: - trigger: step_1 source: '0' From 1b53222937eb75fd1b71fd8283a12be200415614 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 7 Jun 2019 14:41:27 +0200 Subject: [PATCH 21/68] Replace PBKDF2 with scrypt --- solvable/src/webservice/crypto.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/solvable/src/webservice/crypto.py b/solvable/src/webservice/crypto.py index 057bb57..27ed1f5 100644 --- a/solvable/src/webservice/crypto.py +++ b/solvable/src/webservice/crypto.py @@ -1,11 +1,19 @@ -from passlib.hash import pbkdf2_sha256 +from os import urandom +from hashlib import scrypt class PasswordHasher: @staticmethod def hash(password): - return pbkdf2_sha256.hash(password) + salt = urandom(32) + return PasswordHasher.scrypt(password, salt).hex()+salt.hex() @staticmethod def verify(password, hashdigest): - return pbkdf2_sha256.verify(password, hashdigest) + salt = bytes.fromhex(hashdigest[64:]) + hashdigest = bytes.fromhex(hashdigest[:64]) + return PasswordHasher.scrypt(password, salt) == hashdigest + + @staticmethod + def scrypt(password, salt): + return scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32) From 9d9021d01d0ecca2cb23233cbb9b5ff1f73d0164 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 7 Jun 2019 14:42:02 +0200 Subject: [PATCH 22/68] Replace TFWConnector with PipeIO in the webservice --- solvable/Dockerfile | 6 --- solvable/src/webservice/pipe_io.py | 69 +++++++++++++++++++++++++++++ solvable/src/webservice/user_ops.py | 18 ++++---- 3 files changed, 79 insertions(+), 14 deletions(-) create mode 100644 solvable/src/webservice/pipe_io.py diff --git a/solvable/Dockerfile b/solvable/Dockerfile index aa6382f..d61c83a 100644 --- a/solvable/Dockerfile +++ b/solvable/Dockerfile @@ -1,11 +1,5 @@ FROM eu.gcr.io/avatao-challengestore/tutorial-framework -# Install webservice dependencies -RUN pip3 install Flask==1.0 \ - SQLAlchemy==1.2.7 \ - passlib==1.7.1 \ - git+https://github.com/avatao-content/tfwconnector.git#subdirectory=python3 - # Define variables to use later ENV TFW_EHMAIN_DIR="${TFW_DIR}/builtin_event_handlers" \ TFW_WEBSERVICE_DIR="/srv/webservice" \ diff --git a/solvable/src/webservice/pipe_io.py b/solvable/src/webservice/pipe_io.py new file mode 100644 index 0000000..cac23b4 --- /dev/null +++ b/solvable/src/webservice/pipe_io.py @@ -0,0 +1,69 @@ +from typing import Callable + + +class PipeReader: + def __init__(self, pipe_path): + self._pipe = open(pipe_path, 'rb') + self._message_handler = lambda msg: None + + def __enter__(self): + return self + + def __exit__(self, type_, value, traceback): + self.close() + + def close(self): + self._pipe.close() + + @property + def message_handler(self): + return self._message_handler + + @message_handler.setter + def message_handler(self, value): + if not isinstance(value, Callable): + raise ValueError("message_handler must be callable!") + self._message_handler = value + + def run(self): + msg = self.recv_message() + while msg: + self._message_handler(msg) + msg = self.recv_message() + + def recv_message(self): + return self._pipe.readline()[:-1] + + +class PipeWriter: + def __init__(self, pipe_path): + self._pipe = open(pipe_path, 'wb') + + def __enter__(self): + return self + + def __exit__(self, type_, value, traceback): + self.close() + + def close(self): + self._pipe.close() + + def send_message(self, message): + self._pipe.write(message + b'\n') + self._pipe.flush() + + +class PipeIO: + def __init__(self, in_pipe_path, out_pipe_path): + self.reader = PipeReader(in_pipe_path) + self.writer = PipeWriter(out_pipe_path) + + def __enter__(self): + return self + + def __exit__(self, type_, value, traceback): + self.close() + + def close(self): + self.reader.close() + self.writer.close() diff --git a/solvable/src/webservice/user_ops.py b/solvable/src/webservice/user_ops.py index 84831c1..372db7b 100644 --- a/solvable/src/webservice/user_ops.py +++ b/solvable/src/webservice/user_ops.py @@ -1,7 +1,6 @@ -from functools import partial - -from tfwconnector import MessageSender +from json import dumps +from pipe_io import PipeWriter from crypto import PasswordHasher from model import User from errors import InvalidCredentialsError, UserExistsError @@ -12,11 +11,14 @@ class UserOps: self.username = username self.password = password self.db_session = db_session - self.message_sender = MessageSender() - self.log = partial( - self.message_sender.send, - 'Authenticator' - ) + self.pipe = PipeWriter('/tmp/tfw_json_send') + self.log = lambda message: self.pipe.send_message(dumps({ + 'key': 'message', + 'data': { + 'originator': 'Authenticator', + 'message': message + } + }).encode()) def authenticate(self): """ From a1f148f8e001567775012baa64ad306eb7c1c194 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 7 Jun 2019 15:10:29 +0200 Subject: [PATCH 23/68] Rename variable to be more precise --- solvable/src/webservice/crypto.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/solvable/src/webservice/crypto.py b/solvable/src/webservice/crypto.py index 27ed1f5..051ff22 100644 --- a/solvable/src/webservice/crypto.py +++ b/solvable/src/webservice/crypto.py @@ -9,9 +9,9 @@ class PasswordHasher: return PasswordHasher.scrypt(password, salt).hex()+salt.hex() @staticmethod - def verify(password, hashdigest): - salt = bytes.fromhex(hashdigest[64:]) - hashdigest = bytes.fromhex(hashdigest[:64]) + def verify(password, salted_hash): + salt = bytes.fromhex(salted_hash[64:]) + hashdigest = bytes.fromhex(salted_hash[:64]) return PasswordHasher.scrypt(password, salt) == hashdigest @staticmethod From 4a08687ac886af6064a59e8397b2cf2af8964e4d Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 11 Jun 2019 13:22:22 +0200 Subject: [PATCH 24/68] Make indexing more consistent --- solvable/src/webservice/crypto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solvable/src/webservice/crypto.py b/solvable/src/webservice/crypto.py index 051ff22..ff050a4 100644 --- a/solvable/src/webservice/crypto.py +++ b/solvable/src/webservice/crypto.py @@ -10,8 +10,8 @@ class PasswordHasher: @staticmethod def verify(password, salted_hash): - salt = bytes.fromhex(salted_hash[64:]) - hashdigest = bytes.fromhex(salted_hash[:64]) + salt = bytes.fromhex(salted_hash)[32:] + hashdigest = bytes.fromhex(salted_hash)[:32] return PasswordHasher.scrypt(password, salt) == hashdigest @staticmethod From 9faafa7f4971039e0809d1a29398d5cda2dda63a Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 11 Jun 2019 13:29:18 +0200 Subject: [PATCH 25/68] Refactor PasswordHasher --- solvable/src/webservice/crypto.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/solvable/src/webservice/crypto.py b/solvable/src/webservice/crypto.py index ff050a4..8bd034b 100644 --- a/solvable/src/webservice/crypto.py +++ b/solvable/src/webservice/crypto.py @@ -3,17 +3,22 @@ from hashlib import scrypt class PasswordHasher: - @staticmethod - def hash(password): - salt = urandom(32) - return PasswordHasher.scrypt(password, salt).hex()+salt.hex() + n = 16384 + r = 8 + p = 1 + dklen = 32 - @staticmethod - def verify(password, salted_hash): + @classmethod + def hash(cls, password): + salt = urandom(32) + return cls.scrypt(password, salt).hex() + salt.hex() + + @classmethod + def verify(cls, password, salted_hash): salt = bytes.fromhex(salted_hash)[32:] hashdigest = bytes.fromhex(salted_hash)[:32] - return PasswordHasher.scrypt(password, salt) == hashdigest + return cls.scrypt(password, salt) == hashdigest - @staticmethod - def scrypt(password, salt): - return scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32) + @classmethod + def scrypt(cls, password, salt): + return scrypt(password.encode(), salt=salt, n=cls.n, r=cls.r, p=cls.p, dklen=cls.dklen) From 6e8fc588f1ba351fb4195b4424ac6b01610df76c Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 11 Jun 2019 13:41:20 +0200 Subject: [PATCH 26/68] Fix pipe_io_auxlib --- solvable/src/pipe_io_auxlib.py | 16 ++++++++-------- solvable/src/pipe_io_main.py | 6 +++--- solvable/src/webservice/user_ops.py | 10 ++-------- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/solvable/src/pipe_io_auxlib.py b/solvable/src/pipe_io_auxlib.py index a06ce06..9b85fde 100644 --- a/solvable/src/pipe_io_auxlib.py +++ b/solvable/src/pipe_io_auxlib.py @@ -30,7 +30,7 @@ class SignMessagePipeIOEventHandler(PipeIOEventHandlerBase): message = loads(message_bytes) sign_message(self.auth_key, message) if self.forwarding: - self.server_connector.send(message) + self.server_connector.send_message(message) self.pipe_io.send_message(dumps(message).encode()) @@ -77,7 +77,7 @@ class BotPipeIOEventHandler(PipeIOEventHandlerBase): def handle_pipe_event(self, message_bytes): if message_bytes == b"": if self.queue: - self.server_connector.send({ + self.server_connector.send_message({ 'key': 'queueMessages', 'data': { 'messages': self.queue @@ -137,9 +137,9 @@ class DeployPipeIOEventHandler(PipeIOEventHandlerBase): self.expected = False if message_bytes == b'true': - self.server_connector.send(self.onsuccess) + self.server_connector.send_message(self.onsuccess) elif message_bytes == b'false': - self.server_connector.send(self.onerror) + self.server_connector.send_message(self.onerror) else: raise ValueError( f'{self.pipe_io.in_pipe}: Expected "true" or "false".' @@ -178,7 +178,7 @@ class IdePipeIOEventHandler(PipeIOEventHandlerBase): def handle_pipe_event(self, message_bytes): if not self.selected: - self.server_connector.send({ + self.server_connector.send_message({ 'key': 'mirror', 'data': { 'key': 'ide', @@ -189,7 +189,7 @@ class IdePipeIOEventHandler(PipeIOEventHandlerBase): } }) - self.server_connector.send({ + self.server_connector.send_message({ 'key': 'mirror', 'data': { 'key': 'ide', @@ -199,7 +199,7 @@ class IdePipeIOEventHandler(PipeIOEventHandlerBase): } } }) - self.server_connector.send({ + self.server_connector.send_message({ 'key': 'mirror', 'data': { 'key': 'ide', @@ -229,7 +229,7 @@ class FSMPipeIOEventHandler(PipeIOEventHandlerBase): self.pipe_io.send_message(message['data']['current_state'].encode()) def handle_pipe_event(self, message_bytes): - self.server_connector.send({ + self.server_connector.send_message({ 'key': '', 'trigger': message_bytes.decode() }) diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index b15f6a5..a1e9e9a 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -31,8 +31,7 @@ if __name__ == '__main__': json_pipe = PipeIOEventHandler( '', '/tmp/tfw_json_send', - '/tmp/tfw_json_recv', - permissions=0o666 + '/tmp/tfw_json_recv' ) sign_pipe = SignMessagePipeIOEventHandler( @@ -48,7 +47,8 @@ if __name__ == '__main__': bot_pipe = BotPipeIOEventHandler( '/tmp/tfw_bot_send', - '/tmp/tfw_bot_recv' + '/tmp/tfw_bot_recv', + permissions=0o666 ) deploy_pipe = DeployPipeIOEventHandler( diff --git a/solvable/src/webservice/user_ops.py b/solvable/src/webservice/user_ops.py index 372db7b..3517acb 100644 --- a/solvable/src/webservice/user_ops.py +++ b/solvable/src/webservice/user_ops.py @@ -11,14 +11,8 @@ class UserOps: self.username = username self.password = password self.db_session = db_session - self.pipe = PipeWriter('/tmp/tfw_json_send') - self.log = lambda message: self.pipe.send_message(dumps({ - 'key': 'message', - 'data': { - 'originator': 'Authenticator', - 'message': message - } - }).encode()) + self.pipe = PipeWriter('/tmp/tfw_bot_send') + self.log = lambda message: self.pipe.send_message(message.encode()+b"\n") def authenticate(self): """ From 99461141866422f1499f8fecd4fb90c446eab31f Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 11 Jun 2019 17:27:37 +0200 Subject: [PATCH 27/68] Create event handler to monitor bash commands --- solvable/src/event_handler_main.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 5654713..9d48fd6 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -19,13 +19,25 @@ LOG = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) -def cenator(history): +class TerminalCallbackEventHandler(EventHandlerBase): """ - Logs commands executed in terminal to messages. + Logs commands executed in terminal to messages and invokes an + additional callback function to handle special commands. !! Please remove from production code. !! """ - LOG.debug('User executed command: "%s"', history[-1]) - MessageSender().send('JOHN CENA', f'You\'ve executed "{history[-1]}"') + def __init__(self, key, callback): + self.callback = callback + super().__init__(key) + + def handle_event(self, message): + command = message['value'] + self.cenator(command) + self.callback(command) + + @staticmethod + def cenator(command): + LOG.debug('User executed command: "%s"', command) + MessageSender().send('JOHN CENA', f'You\'ve executed "{command}"') class TestCommands(TerminalCommands): @@ -123,6 +135,10 @@ def main(): key='shell', monitor=BashMonitor(TFWENV.HISTFILE) ) + commands = TerminalCallbackEventHandler( # Reacts to terminal commands + 'history.bash', + TestCommands(bashrc=f'/home/{TAOENV.USER}/.bashrc').callback + ) processmanager = ProcessManagingEventHandler( # Handles 'deploy' button clicks key='processmanager', dirmonitor=ide.monitor, @@ -147,11 +163,6 @@ def main(): key='test' ) - # Terminal command handlers - commands = TestCommands(bashrc=f'/home/{TAOENV.USER}/.bashrc') - terminal.historymonitor.subscribe_callback(commands.callback) - terminal.historymonitor.subscribe_callback(cenator) - event_handlers = EventHandlerBase.get_local_instances() def stop(sig, frame): for eh in event_handlers: From 096bba83670694f1bf2b0e194623fc8c67057aa1 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 18 Jun 2019 18:51:05 +0200 Subject: [PATCH 28/68] Import new logger --- solvable/src/event_handler_main.py | 5 +++-- solvable/supervisor/event_handler_main.conf | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 9d48fd6..16bdd81 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -14,9 +14,9 @@ from tfw.components import FSMManagingEventHandler, DirectorySnapshottingEventHa from tfw.components import FrontendEventHandler, MessageSender from tfw.config import TFWENV from tao.config import TAOENV +from tfw.config.log import TFWLog LOG = logging.getLogger(__name__) -logging.basicConfig(level=logging.DEBUG) class TerminalCallbackEventHandler(EventHandlerBase): @@ -115,7 +115,8 @@ class MessageFSMStepsEventHandler(FSMAwareEventHandler): def main(): # pylint: disable=unused-variable - # + TFWLog().start() + # TFW component EventHandlers (builtins, required for their respective functionalities) fsm = FSMManagingEventHandler( # TFW FSM key='fsm', diff --git a/solvable/supervisor/event_handler_main.conf b/solvable/supervisor/event_handler_main.conf index 021e13c..2d89a79 100644 --- a/solvable/supervisor/event_handler_main.conf +++ b/solvable/supervisor/event_handler_main.conf @@ -2,3 +2,6 @@ user=root directory=%(ENV_TFW_EHMAIN_DIR)s command=python3 event_handler_main.py + +[supervisord] +strip_ansi=false From dc42e0f4d00d2ac9a7a137b687c8e80b20eb8a3a Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 20 Jun 2019 16:06:23 +0200 Subject: [PATCH 29/68] Separate event handlers from independent components --- solvable/src/event_handler_main.py | 10 +++++----- solvable/src/pipe_io_auxlib.py | 9 +++++---- solvable/src/pipe_io_main.py | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 16bdd81..94843c9 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -7,11 +7,11 @@ from tornado.ioloop import IOLoop from tfw.fsm import YamlFSM from tfw.event_handlers import EventHandlerBase, FSMAwareEventHandler, TFWServerUplinkConnector -from tfw.components import IdeEventHandler, TerminalEventHandler -from tfw.components import ProcessManagingEventHandler, BashMonitor -from tfw.components import TerminalCommands, LogMonitoringEventHandler -from tfw.components import FSMManagingEventHandler, DirectorySnapshottingEventHandler -from tfw.components import FrontendEventHandler, MessageSender +from tfw.builtins import FrontendEventHandler +from tfw.builtins import IdeEventHandler, TerminalEventHandler +from tfw.builtins import LogMonitoringEventHandler, ProcessManagingEventHandler +from tfw.builtins import DirectorySnapshottingEventHandler, FSMManagingEventHandler +from tfw.components import BashMonitor, MessageSender, TerminalCommands from tfw.config import TFWENV from tao.config import TAOENV from tfw.config.log import TFWLog diff --git a/solvable/src/pipe_io_auxlib.py b/solvable/src/pipe_io_auxlib.py index 9b85fde..d866fe0 100644 --- a/solvable/src/pipe_io_auxlib.py +++ b/solvable/src/pipe_io_auxlib.py @@ -1,8 +1,9 @@ from json import dumps, loads from tfw.crypto import KeyManager, sign_message, verify_message -from tfw.components import PipeIOEventHandlerBase -from tfw.components.pipe_io_event_handler import DEFAULT_PERMISSIONS +from tfw.builtins import PipeIOEventHandlerBase +from tfw.builtins.pipe_io_event_handler import DEFAULT_PERMISSIONS +from tfw.networking.scope import Scope class SignMessagePipeIOEventHandler(PipeIOEventHandlerBase): @@ -137,9 +138,9 @@ class DeployPipeIOEventHandler(PipeIOEventHandlerBase): self.expected = False if message_bytes == b'true': - self.server_connector.send_message(self.onsuccess) + self.server_connector.send_message(self.onsuccess, scope=Scope.WEBSOCKET) elif message_bytes == b'false': - self.server_connector.send_message(self.onerror) + self.server_connector.send_message(self.onerror, scope=Scope.WEBSOCKET) else: raise ValueError( f'{self.pipe_io.in_pipe}: Expected "true" or "false".' diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index a1e9e9a..b446820 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -4,7 +4,7 @@ from signal import signal, SIGTERM, SIGINT from tornado.ioloop import IOLoop from tfw.event_handlers import EventHandlerBase -from tfw.components import PipeIOEventHandler +from tfw.builtins import PipeIOEventHandler from pipe_io_auxlib import ( SignMessagePipeIOEventHandler, VerifyMessagePipeIOEventHandler, From 323300a676d8ead302c90182ae9070403977292b Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 20 Jun 2019 16:07:26 +0200 Subject: [PATCH 30/68] Add new logger to PipeIO --- solvable/src/pipe_io_main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index b446820..42bbe8d 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -5,6 +5,7 @@ from tornado.ioloop import IOLoop from tfw.event_handlers import EventHandlerBase from tfw.builtins import PipeIOEventHandler +from tfw.config.log import TFWLog from pipe_io_auxlib import ( SignMessagePipeIOEventHandler, VerifyMessagePipeIOEventHandler, @@ -13,7 +14,7 @@ from pipe_io_auxlib import ( FSMPipeIOEventHandler ) -logging.basicConfig(level=logging.DEBUG) +LOG = logging.getLogger(__name__) if __name__ == '__main__': From 957f2c9477681fae5b7babc8e5c5955a1f5834eb Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 24 Jun 2019 14:30:52 +0200 Subject: [PATCH 31/68] Remove dirmonitor --- solvable/src/event_handler_main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 94843c9..ef48f53 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -142,7 +142,6 @@ def main(): ) processmanager = ProcessManagingEventHandler( # Handles 'deploy' button clicks key='processmanager', - dirmonitor=ide.monitor, log_tail=2000 ) logmonitor = LogMonitoringEventHandler( # Sends live logs of webservice process to frontend From 8335338a9648a2c066f4f59534547f4fa6001a68 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 27 Jun 2019 14:38:36 +0200 Subject: [PATCH 32/68] Fix pylint warnings --- solvable/src/event_handler_main.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index ef48f53..724e940 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -10,11 +10,11 @@ from tfw.event_handlers import EventHandlerBase, FSMAwareEventHandler, TFWServer from tfw.builtins import FrontendEventHandler from tfw.builtins import IdeEventHandler, TerminalEventHandler from tfw.builtins import LogMonitoringEventHandler, ProcessManagingEventHandler -from tfw.builtins import DirectorySnapshottingEventHandler, FSMManagingEventHandler -from tfw.components import BashMonitor, MessageSender, TerminalCommands +from tfw.builtins import DirectorySnapshottingEventHandler, FSMManagingEventHandler, MessageSender +from tfw.components import TerminalCommands from tfw.config import TFWENV -from tao.config import TAOENV from tfw.config.log import TFWLog +from tao.config import TAOENV LOG = logging.getLogger(__name__) @@ -133,8 +133,7 @@ def main(): exclude=['*.pyc'] ) terminal = TerminalEventHandler( # Web shell backend - key='shell', - monitor=BashMonitor(TFWENV.HISTFILE) + key='shell' ) commands = TerminalCallbackEventHandler( # Reacts to terminal commands 'history.bash', From b3927496432b83c9c62c234576ae82d75df342eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 28 Jun 2019 15:12:27 +0200 Subject: [PATCH 33/68] Comply EventHandler changes --- .pylintrc | 2 +- controller/opt/server.py | 2 +- solvable/src/event_handler_main.py | 11 +++++------ solvable/src/pipe_io_auxlib.py | 1 - solvable/src/pipe_io_main.py | 4 +++- solvable/src/test_fsm.py | 2 +- solvable/src/webservice/user_ops.py | 2 -- 7 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.pylintrc b/.pylintrc index 1cbae5d..961a526 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,7 +6,7 @@ disable = missing-docstring, too-few-public-methods, invalid-name [SIMILARITIES] -min-similarity-lines=6 +min-similarity-lines=7 ignore-comments=yes ignore-docstrings=yes ignore-imports=yes diff --git a/controller/opt/server.py b/controller/opt/server.py index 9e211c5..c34f0a2 100644 --- a/controller/opt/server.py +++ b/controller/opt/server.py @@ -4,7 +4,7 @@ import json from tornado.ioloop import IOLoop from tornado.web import RequestHandler, Application -from tfw.event_handlers import FSMAwareEventHandler +from tfw.builtins import FSMAwareEventHandler class ControllerPostHandler(RequestHandler): diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 724e940..1d05af4 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -6,9 +6,8 @@ from signal import signal, SIGTERM, SIGINT from tornado.ioloop import IOLoop from tfw.fsm import YamlFSM -from tfw.event_handlers import EventHandlerBase, FSMAwareEventHandler, TFWServerUplinkConnector -from tfw.builtins import FrontendEventHandler -from tfw.builtins import IdeEventHandler, TerminalEventHandler +from tfw.builtins import EventHandler, FSMAwareEventHandler, TFWServerUplinkConnector +from tfw.builtins import IdeEventHandler, TerminalEventHandler, FrontendEventHandler from tfw.builtins import LogMonitoringEventHandler, ProcessManagingEventHandler from tfw.builtins import DirectorySnapshottingEventHandler, FSMManagingEventHandler, MessageSender from tfw.components import TerminalCommands @@ -19,7 +18,7 @@ from tao.config import TAOENV LOG = logging.getLogger(__name__) -class TerminalCallbackEventHandler(EventHandlerBase): +class TerminalCallbackEventHandler(EventHandler): """ Logs commands executed in terminal to messages and invokes an additional callback function to handle special commands. @@ -162,8 +161,8 @@ def main(): key='test' ) - event_handlers = EventHandlerBase.get_local_instances() - def stop(sig, frame): + event_handlers = EventHandler.get_local_instances() + def stop(sig, frame): # pylint: disable=unused-argument for eh in event_handlers: eh.stop() exit(0) diff --git a/solvable/src/pipe_io_auxlib.py b/solvable/src/pipe_io_auxlib.py index d866fe0..933a594 100644 --- a/solvable/src/pipe_io_auxlib.py +++ b/solvable/src/pipe_io_auxlib.py @@ -14,7 +14,6 @@ class SignMessagePipeIOEventHandler(PipeIOEventHandlerBase): When forwarding is true, it will send the signed message to the TFW server before writing it into the output pipe. """ - def __init__( self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS, diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index 42bbe8d..d737668 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -18,6 +18,8 @@ LOG = logging.getLogger(__name__) if __name__ == '__main__': + # pylint: disable=pointless-string-statement + TFWLog().start() """ Creates general purpose pipes. The first parameter associates the receiving pipe with a key, which is @@ -71,7 +73,7 @@ if __name__ == '__main__': ) event_handlers = EventHandlerBase.get_local_instances() - def stop(sig, frame): + def stop(sig, frame): # pylint: disable=unused-argument for eh in event_handlers: eh.stop() exit(0) diff --git a/solvable/src/test_fsm.py b/solvable/src/test_fsm.py index 2db9b97..9170e90 100644 --- a/solvable/src/test_fsm.py +++ b/solvable/src/test_fsm.py @@ -3,7 +3,7 @@ from os.path import exists from tfw.fsm import LinearFSM -from tfw.networking import MessageSender +from tfw.builtins import MessageSender class TestFSM(LinearFSM): diff --git a/solvable/src/webservice/user_ops.py b/solvable/src/webservice/user_ops.py index 3517acb..66855e8 100644 --- a/solvable/src/webservice/user_ops.py +++ b/solvable/src/webservice/user_ops.py @@ -1,5 +1,3 @@ -from json import dumps - from pipe_io import PipeWriter from crypto import PasswordHasher from model import User From 8590aba8dd22ff52ba26847728003000de4366eb Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 28 Jun 2019 17:36:03 +0200 Subject: [PATCH 34/68] Refactor TFW logging --- solvable/src/event_handler_main.py | 8 ++++++-- solvable/src/pipe_io_main.py | 11 ++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 1d05af4..e9c757a 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -1,4 +1,5 @@ import logging +from sys import stderr from ast import literal_eval from functools import partial from signal import signal, SIGTERM, SIGINT @@ -12,8 +13,8 @@ from tfw.builtins import LogMonitoringEventHandler, ProcessManagingEventHandler from tfw.builtins import DirectorySnapshottingEventHandler, FSMManagingEventHandler, MessageSender from tfw.components import TerminalCommands from tfw.config import TFWENV -from tfw.config.log import TFWLog from tao.config import TAOENV +from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter LOG = logging.getLogger(__name__) @@ -114,7 +115,10 @@ class MessageFSMStepsEventHandler(FSMAwareEventHandler): def main(): # pylint: disable=unused-variable - TFWLog().start() + Logger([ + Log(stderr, LogFormatter(20)), + Log(TFWENV.LOGFILE, VerboseLogFormatter()) + ]).start() # TFW component EventHandlers (builtins, required for their respective functionalities) fsm = FSMManagingEventHandler( # TFW FSM diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index d737668..fabd772 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -1,11 +1,13 @@ import logging +from sys import stderr from signal import signal, SIGTERM, SIGINT from tornado.ioloop import IOLoop from tfw.event_handlers import EventHandlerBase from tfw.builtins import PipeIOEventHandler -from tfw.config.log import TFWLog +from tfw.config import TFWENV +from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter from pipe_io_auxlib import ( SignMessagePipeIOEventHandler, VerifyMessagePipeIOEventHandler, @@ -18,8 +20,11 @@ LOG = logging.getLogger(__name__) if __name__ == '__main__': - # pylint: disable=pointless-string-statement - TFWLog().start() + Logger([ + Log(stderr, LogFormatter(20)), + Log(TFWENV.LOGFILE, VerboseLogFormatter()) + ]).start() + """ Creates general purpose pipes. The first parameter associates the receiving pipe with a key, which is From d263011179f3c7304dfbda6aa162f55cf80dfb87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 1 Jul 2019 14:54:47 +0200 Subject: [PATCH 35/68] Fix import order --- solvable/src/event_handler_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index e9c757a..64fc82a 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -13,8 +13,8 @@ from tfw.builtins import LogMonitoringEventHandler, ProcessManagingEventHandler from tfw.builtins import DirectorySnapshottingEventHandler, FSMManagingEventHandler, MessageSender from tfw.components import TerminalCommands from tfw.config import TFWENV -from tao.config import TAOENV from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter +from tao.config import TAOENV LOG = logging.getLogger(__name__) From 76118c0c4902e200ea2bacaa53edfd7f47809eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 1 Jul 2019 16:02:44 +0200 Subject: [PATCH 36/68] Use new EventHandlerBase API to simplify graceful stop logic --- solvable/src/event_handler_main.py | 6 ++---- solvable/src/pipe_io_main.py | 26 +++++++++----------------- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 64fc82a..2b5ae62 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -165,10 +165,8 @@ def main(): key='test' ) - event_handlers = EventHandler.get_local_instances() - def stop(sig, frame): # pylint: disable=unused-argument - for eh in event_handlers: - eh.stop() + def stop(*_): + EventHandler.stop_all_instances() exit(0) signal(SIGTERM, stop) signal(SIGINT, stop) diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index fabd772..1e9e26b 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -4,7 +4,7 @@ from signal import signal, SIGTERM, SIGINT from tornado.ioloop import IOLoop -from tfw.event_handlers import EventHandlerBase +from tfw.builtins import EventHandler from tfw.builtins import PipeIOEventHandler from tfw.config import TFWENV from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter @@ -19,23 +19,13 @@ from pipe_io_auxlib import ( LOG = logging.getLogger(__name__) -if __name__ == '__main__': +def main(): + # pylint: disable=unused-variable Logger([ Log(stderr, LogFormatter(20)), Log(TFWENV.LOGFILE, VerboseLogFormatter()) ]).start() - """ - Creates general purpose pipes. - The first parameter associates the receiving pipe with a key, which is - an empty string in this case. It has a special meaning, you can - subscribe to every kind of message with this key. - If you wish to filter incoming data, specify a single or more keys in - a list, eg.: processmanager, ide, key... - You can send/receive JSON messages to/from the TFW server as any user, - because we gave read+write permissions, without that parameter, only - the owner has access to the pipes. - """ json_pipe = PipeIOEventHandler( '', '/tmp/tfw_json_send', @@ -77,12 +67,14 @@ if __name__ == '__main__': '/tmp/tfw_fsm_recv' ) - event_handlers = EventHandlerBase.get_local_instances() - def stop(sig, frame): # pylint: disable=unused-argument - for eh in event_handlers: - eh.stop() + def stop(*_): + EventHandler.stop_all_instances() exit(0) signal(SIGTERM, stop) signal(SIGINT, stop) IOLoop.instance().start() + + +if __name__ == '__main__': + main() From d597e4e00418697f4436e5b8d810000e9402efc0 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 2 Jul 2019 15:37:12 +0200 Subject: [PATCH 37/68] Run Python in unbuffered mode --- solvable/supervisor/event_handler_main.conf | 2 +- solvable/supervisor/pipe_io_main.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/solvable/supervisor/event_handler_main.conf b/solvable/supervisor/event_handler_main.conf index 2d89a79..60112be 100644 --- a/solvable/supervisor/event_handler_main.conf +++ b/solvable/supervisor/event_handler_main.conf @@ -1,7 +1,7 @@ [program:event_handler_main] user=root directory=%(ENV_TFW_EHMAIN_DIR)s -command=python3 event_handler_main.py +command=python3 -u event_handler_main.py [supervisord] strip_ansi=false diff --git a/solvable/supervisor/pipe_io_main.conf b/solvable/supervisor/pipe_io_main.conf index bdf8a1a..6da22c2 100644 --- a/solvable/supervisor/pipe_io_main.conf +++ b/solvable/supervisor/pipe_io_main.conf @@ -1,4 +1,4 @@ [program:pipe_io_main] user=root directory=%(ENV_TFW_EHMAIN_DIR)s -command=python3 pipe_io_main.py +command=python3 -u pipe_io_main.py From 417a848328af1ac157faf2bb0c109e7364da7ecc Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 5 Jul 2019 15:37:54 +0200 Subject: [PATCH 38/68] Move custom event handlers away --- solvable/src/custom_event_handlers.py | 100 ++++++++++++++++++++++++ solvable/src/event_handler_main.py | 105 ++------------------------ solvable/src/test_fsm.py | 6 +- 3 files changed, 110 insertions(+), 101 deletions(-) create mode 100644 solvable/src/custom_event_handlers.py diff --git a/solvable/src/custom_event_handlers.py b/solvable/src/custom_event_handlers.py new file mode 100644 index 0000000..c5d0a33 --- /dev/null +++ b/solvable/src/custom_event_handlers.py @@ -0,0 +1,100 @@ +import logging +from ast import literal_eval + +from tfw.components import MessageSender, TerminalCommands +from tfw.builtins import EventHandler, FSMAwareEventHandler, TFWServerUplinkConnector + +LOG = logging.getLogger(__name__) + + +class TerminalCallbackEventHandler(EventHandler): + """ + Logs commands executed in terminal to messages and invokes an + additional callback function to handle special commands. + !! Please remove from production code. !! + """ + def __init__(self, key, callback): + self.callback = callback + super().__init__(key) + + def handle_event(self, message): + command = message['value'] + self.cenator(command) + self.callback(command) + + def cenator(self, command): + LOG.debug('User executed command: "%s"', command) + MessageSender(self.server_connector).send('JOHN CENA', f'You\'ve executed "{command}"') + + +class TestCommands(TerminalCommands): + """ + Some example commands useful for debugging. + !! Please remove from production code !! and inherit your own + class from TerminalCommands if you need to define custom + commands in your challenge. + """ + # pylint: disable=unused-argument, attribute-defined-outside-init, no-self-use + def command_sendmessage(self, *args): + """ + Insert TFW message template as first argument if executed without args. + + Evaluate first argumen as a dict and send it to the frontend. + This is useful for playing around with frontend APIs. + """ + if not args: + message_template = """'{"key": "", "data": {"command": ""}}'""" + TFWServerUplinkConnector().send_message({ + 'key': 'shell', + 'data': { + 'command': 'write', + 'value': f'sendmessage {message_template}' + } + }) + else: + TFWServerUplinkConnector().send_message(literal_eval(args[0])) + + def command_seppuku_tfw(self, *args): + """ + Restart tfw_server.py and event_handler_main.py. + This can speed up development when combined with mounting + volumes from host to container. + """ + seppuku = ( + 'nohup sh -c "supervisorctl restart tfwserver event_handler_main" &> /dev/null & ' + 'clear && echo "Committed seppuku! :)" && sleep infinity' + ) + uplink = TFWServerUplinkConnector() + uplink.send_message({ + 'key': 'shell', + 'data': { + 'command': 'write', + 'value': f'{seppuku}\n' + } + }) + uplink.send_message({ + 'key': 'dashboard', + 'data': { + 'command': 'reloadFrontend' + } + }) + + +class MessageFSMStepsEventHandler(FSMAwareEventHandler): + """ + This example EventHandler is capable of detecting FSM state. + !! Please remove from production code !! + """ + def handle_event(self, message): + pass + + def handle_fsm_step(self, **kwargs): + """ + When the FSM steps this method is invoked. + Receives a 'data' field from an fsm_update message as kwargs. + """ + MessageSender(self.server_connector).send( + 'FSM info', + f'FSM has stepped from state "{kwargs["last_event"]["from_state"]}" ' + f'to state "{kwargs["current_state"]}" in response to trigger "{kwargs["last_event"]["trigger"]}"' + ) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 2b5ae62..bbde28a 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -1,118 +1,25 @@ import logging from sys import stderr -from ast import literal_eval from functools import partial from signal import signal, SIGTERM, SIGINT from tornado.ioloop import IOLoop from tfw.fsm import YamlFSM -from tfw.builtins import EventHandler, FSMAwareEventHandler, TFWServerUplinkConnector +from tfw.builtins import EventHandler from tfw.builtins import IdeEventHandler, TerminalEventHandler, FrontendEventHandler from tfw.builtins import LogMonitoringEventHandler, ProcessManagingEventHandler -from tfw.builtins import DirectorySnapshottingEventHandler, FSMManagingEventHandler, MessageSender -from tfw.components import TerminalCommands +from tfw.builtins import DirectorySnapshottingEventHandler, FSMManagingEventHandler from tfw.config import TFWENV from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter from tao.config import TAOENV +from custom_event_handlers import MessageFSMStepsEventHandler +from custom_event_handlers import TerminalCallbackEventHandler, TestCommands + LOG = logging.getLogger(__name__) -class TerminalCallbackEventHandler(EventHandler): - """ - Logs commands executed in terminal to messages and invokes an - additional callback function to handle special commands. - !! Please remove from production code. !! - """ - def __init__(self, key, callback): - self.callback = callback - super().__init__(key) - - def handle_event(self, message): - command = message['value'] - self.cenator(command) - self.callback(command) - - @staticmethod - def cenator(command): - LOG.debug('User executed command: "%s"', command) - MessageSender().send('JOHN CENA', f'You\'ve executed "{command}"') - - -class TestCommands(TerminalCommands): - """ - Some example commands useful for debugging. - !! Please remove from production code !! and inherit your own - class from TerminalCommands if you need to define custom - commands in your challenge. - """ - # pylint: disable=unused-argument, attribute-defined-outside-init, no-self-use - def command_sendmessage(self, *args): - """ - Insert TFW message template as first argument if executed without args. - - Evaluate first argumen as a dict and send it to the frontend. - This is useful for playing around with frontend APIs. - """ - if not args: - message_template = """'{"key": "", "data": {"command": ""}}'""" - TFWServerUplinkConnector().send_message({ - 'key': 'shell', - 'data': { - 'command': 'write', - 'value': f'sendmessage {message_template}' - } - }) - else: - TFWServerUplinkConnector().send_message(literal_eval(args[0])) - - def command_seppuku_tfw(self, *args): - """ - Restart tfw_server.py and event_handler_main.py. - This can speed up development when combined with mounting - volumes from host to container. - """ - seppuku = ( - 'nohup sh -c "supervisorctl restart tfwserver event_handler_main" &> /dev/null & ' - 'clear && echo "Committed seppuku! :)" && sleep infinity' - ) - uplink = TFWServerUplinkConnector() - uplink.send_message({ - 'key': 'shell', - 'data': { - 'command': 'write', - 'value': f'{seppuku}\n' - } - }) - uplink.send_message({ - 'key': 'dashboard', - 'data': { - 'command': 'reloadFrontend' - } - }) - - -class MessageFSMStepsEventHandler(FSMAwareEventHandler): - """ - This example EventHandler is capable of detecting FSM state. - !! Please remove from production code !! - """ - def handle_event(self, message): - pass - - def handle_fsm_step(self, **kwargs): - """ - When the FSM steps this method is invoked. - Receives a 'data' field from an fsm_update message as kwargs. - """ - MessageSender().send( - 'FSM info', - f'FSM has stepped from state "{kwargs["last_event"]["from_state"]}" ' - f'to state "{kwargs["current_state"]}" in response to trigger "{kwargs["last_event"]["trigger"]}"' - ) - - def main(): # pylint: disable=unused-variable Logger([ @@ -165,7 +72,7 @@ def main(): key='test' ) - def stop(*_): + def stop(*_): EventHandler.stop_all_instances() exit(0) signal(SIGTERM, stop) diff --git a/solvable/src/test_fsm.py b/solvable/src/test_fsm.py index 9170e90..f4f7639 100644 --- a/solvable/src/test_fsm.py +++ b/solvable/src/test_fsm.py @@ -3,7 +3,8 @@ from os.path import exists from tfw.fsm import LinearFSM -from tfw.builtins import MessageSender +from tfw.components import MessageSender +from tfw.builtins import TFWServerUplinkConnector class TestFSM(LinearFSM): @@ -11,7 +12,8 @@ class TestFSM(LinearFSM): def __init__(self): super().__init__(6) - self.message_sender = MessageSender() + self.uplink = TFWServerUplinkConnector() + self.message_sender = MessageSender(self.uplink) self.subscribe_predicate('step_3', self.step_3_allowed) @staticmethod From ef2ab5d0bf7390bb0881d5d477efeb538e37c63e Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 8 Jul 2019 11:42:03 +0200 Subject: [PATCH 39/68] Move signal handling to new module --- solvable/src/event_handler_main.py | 10 ++-------- solvable/src/pipe_io_main.py | 10 ++-------- solvable/src/signal_handling.py | 11 +++++++++++ 3 files changed, 15 insertions(+), 16 deletions(-) create mode 100644 solvable/src/signal_handling.py diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index bbde28a..4a18323 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -1,12 +1,10 @@ import logging from sys import stderr from functools import partial -from signal import signal, SIGTERM, SIGINT from tornado.ioloop import IOLoop from tfw.fsm import YamlFSM -from tfw.builtins import EventHandler from tfw.builtins import IdeEventHandler, TerminalEventHandler, FrontendEventHandler from tfw.builtins import LogMonitoringEventHandler, ProcessManagingEventHandler from tfw.builtins import DirectorySnapshottingEventHandler, FSMManagingEventHandler @@ -16,6 +14,7 @@ from tao.config import TAOENV from custom_event_handlers import MessageFSMStepsEventHandler from custom_event_handlers import TerminalCallbackEventHandler, TestCommands +from signal_handling import setup_signal_handlers LOG = logging.getLogger(__name__) @@ -72,12 +71,7 @@ def main(): key='test' ) - def stop(*_): - EventHandler.stop_all_instances() - exit(0) - signal(SIGTERM, stop) - signal(SIGINT, stop) - + setup_signal_handlers() IOLoop.instance().start() diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index 1e9e26b..27d886f 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -1,10 +1,8 @@ import logging from sys import stderr -from signal import signal, SIGTERM, SIGINT from tornado.ioloop import IOLoop -from tfw.builtins import EventHandler from tfw.builtins import PipeIOEventHandler from tfw.config import TFWENV from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter @@ -15,6 +13,7 @@ from pipe_io_auxlib import ( DeployPipeIOEventHandler, IdePipeIOEventHandler, FSMPipeIOEventHandler ) +from signal_handling import setup_signal_handlers LOG = logging.getLogger(__name__) @@ -67,12 +66,7 @@ def main(): '/tmp/tfw_fsm_recv' ) - def stop(*_): - EventHandler.stop_all_instances() - exit(0) - signal(SIGTERM, stop) - signal(SIGINT, stop) - + setup_signal_handlers() IOLoop.instance().start() diff --git a/solvable/src/signal_handling.py b/solvable/src/signal_handling.py new file mode 100644 index 0000000..21294d2 --- /dev/null +++ b/solvable/src/signal_handling.py @@ -0,0 +1,11 @@ +from signal import signal, SIGTERM, SIGINT + +from tfw.builtins import EventHandler + + +def setup_signal_handlers(): + def stop(*_): + EventHandler.stop_all_instances() + exit(0) + signal(SIGTERM, stop) + signal(SIGINT, stop) From e50a8732fc70c3596376c01b464205a1a06537e0 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 8 Jul 2019 14:12:21 +0200 Subject: [PATCH 40/68] Refactor TerminalCommands --- solvable/src/custom_event_handlers.py | 41 ++++----------------------- solvable/src/event_handler_main.py | 9 +++--- 2 files changed, 11 insertions(+), 39 deletions(-) diff --git a/solvable/src/custom_event_handlers.py b/solvable/src/custom_event_handlers.py index c5d0a33..ca9749b 100644 --- a/solvable/src/custom_event_handlers.py +++ b/solvable/src/custom_event_handlers.py @@ -1,33 +1,29 @@ import logging from ast import literal_eval -from tfw.components import MessageSender, TerminalCommands -from tfw.builtins import EventHandler, FSMAwareEventHandler, TFWServerUplinkConnector +from tfw.components import MessageSender +from tfw.builtins import TFWServerUplinkConnector +from tfw.builtins import EventHandler, FSMAwareEventHandler, TerminalCommandsEventHandler LOG = logging.getLogger(__name__) -class TerminalCallbackEventHandler(EventHandler): +class CenatorEventHandler(EventHandler): """ Logs commands executed in terminal to messages and invokes an additional callback function to handle special commands. !! Please remove from production code. !! """ - def __init__(self, key, callback): - self.callback = callback + def __init__(self, key): super().__init__(key) def handle_event(self, message): command = message['value'] - self.cenator(command) - self.callback(command) - - def cenator(self, command): LOG.debug('User executed command: "%s"', command) MessageSender(self.server_connector).send('JOHN CENA', f'You\'ve executed "{command}"') -class TestCommands(TerminalCommands): +class TestCommandsEventHandler(TerminalCommandsEventHandler): """ Some example commands useful for debugging. !! Please remove from production code !! and inherit your own @@ -54,31 +50,6 @@ class TestCommands(TerminalCommands): else: TFWServerUplinkConnector().send_message(literal_eval(args[0])) - def command_seppuku_tfw(self, *args): - """ - Restart tfw_server.py and event_handler_main.py. - This can speed up development when combined with mounting - volumes from host to container. - """ - seppuku = ( - 'nohup sh -c "supervisorctl restart tfwserver event_handler_main" &> /dev/null & ' - 'clear && echo "Committed seppuku! :)" && sleep infinity' - ) - uplink = TFWServerUplinkConnector() - uplink.send_message({ - 'key': 'shell', - 'data': { - 'command': 'write', - 'value': f'{seppuku}\n' - } - }) - uplink.send_message({ - 'key': 'dashboard', - 'data': { - 'command': 'reloadFrontend' - } - }) - class MessageFSMStepsEventHandler(FSMAwareEventHandler): """ diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 4a18323..b31bd2d 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -13,7 +13,7 @@ from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter from tao.config import TAOENV from custom_event_handlers import MessageFSMStepsEventHandler -from custom_event_handlers import TerminalCallbackEventHandler, TestCommands +from custom_event_handlers import CenatorEventHandler, TestCommandsEventHandler from signal_handling import setup_signal_handlers LOG = logging.getLogger(__name__) @@ -44,9 +44,10 @@ def main(): terminal = TerminalEventHandler( # Web shell backend key='shell' ) - commands = TerminalCallbackEventHandler( # Reacts to terminal commands - 'history.bash', - TestCommands(bashrc=f'/home/{TAOENV.USER}/.bashrc').callback + cenator = CenatorEventHandler('history.bash') # Reacts to terminal commands + commands = TestCommandsEventHandler( # Catches special commands + key='history.bash', + bashrc=f'/home/{TAOENV.USER}/.bashrc' ) processmanager = ProcessManagingEventHandler( # Handles 'deploy' button clicks key='processmanager', From eb60c1e08ed5e31ae1444d41fff4eaf5893c193f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 12 Jul 2019 23:19:22 +0200 Subject: [PATCH 41/68] Remove pipe auxlib --- solvable/src/pipe_io_auxlib.py | 235 --------------------------------- 1 file changed, 235 deletions(-) delete mode 100644 solvable/src/pipe_io_auxlib.py diff --git a/solvable/src/pipe_io_auxlib.py b/solvable/src/pipe_io_auxlib.py deleted file mode 100644 index 933a594..0000000 --- a/solvable/src/pipe_io_auxlib.py +++ /dev/null @@ -1,235 +0,0 @@ -from json import dumps, loads - -from tfw.crypto import KeyManager, sign_message, verify_message -from tfw.builtins import PipeIOEventHandlerBase -from tfw.builtins.pipe_io_event_handler import DEFAULT_PERMISSIONS -from tfw.networking.scope import Scope - - -class SignMessagePipeIOEventHandler(PipeIOEventHandlerBase): - """ - Signs a valid TFW message with HMAC. - Note that the running process needs root permissions in order to read - the authentication key. - When forwarding is true, it will send the signed message to the TFW - server before writing it into the output pipe. - """ - def __init__( - self, in_pipe_path, out_pipe_path, - permissions=DEFAULT_PERMISSIONS, - forwarding=True - ): - self.forwarding = forwarding - self.auth_key = KeyManager().auth_key - super().__init__(None, in_pipe_path, out_pipe_path, permissions) - - def handle_event(self, message): - pass - - def handle_pipe_event(self, message_bytes): - message = loads(message_bytes) - sign_message(self.auth_key, message) - if self.forwarding: - self.server_connector.send_message(message) - self.pipe_io.send_message(dumps(message).encode()) - - -class VerifyMessagePipeIOEventHandler(PipeIOEventHandlerBase): - """ - Verifies a signed TFW message. - This pipe also needs root permissions. Send the serialized JSON object - to the pipe, then wait for its boolean response. - """ - - def __init__(self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS): - self.auth_key = KeyManager().auth_key - super().__init__(None, in_pipe_path, out_pipe_path, permissions) - - def handle_event(self, message): - pass - - def handle_pipe_event(self, message_bytes): - message = loads(message_bytes) - validity = verify_message(self.auth_key, message) - self.pipe_io.send_message(str(validity).lower().encode()) - - -class BotPipeIOEventHandler(PipeIOEventHandlerBase): - """ - Sends bot messages to the frontend. - If you assign @originator, it will be the default message sender. - When you write a line to the pipe, it will be considered as a single - message and gets appended to the queue until an empty line is received, - which triggers forwarding the messages to the TFW server. - """ - - def __init__( - self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS, - originator='avataobot' - ): - self.queue = [] - self.originator = originator - super().__init__(None, in_pipe_path, out_pipe_path, permissions) - - def handle_event(self, message): - pass - - def handle_pipe_event(self, message_bytes): - if message_bytes == b"": - if self.queue: - self.server_connector.send_message({ - 'key': 'queueMessages', - 'data': { - 'messages': self.queue - } - }) - self.queue = [] - else: - self.queue.append({ - 'originator': self.originator, - 'message': message_bytes.decode().replace('\\n', '\n') - }) - - -class DeployPipeIOEventHandler(PipeIOEventHandlerBase): - """ - Manages deployment in the IDE. - When you receive "deploy", then you have to answer with a "true" or - "false" depending whether you are satisfied with the result or not. - The @process parameter is the name of the supervised service. - """ - - # pylint: disable=too-many-arguments - def __init__( - self, in_pipe_path, out_pipe_path, process, - permissions=DEFAULT_PERMISSIONS - ): - self.expected = False - self.process = process - - self.onsuccess = { - 'key': 'processmanager', - 'data': { - 'process_name': process, - 'command': 'restart' - } - } - self.onerror = { - 'key': 'processmanager', - 'data': { - 'process_name': process, - 'error': True - } - } - - super().__init__('processmanager', in_pipe_path, out_pipe_path, permissions) - - def handle_event(self, message): - if message == self.onsuccess: - self.expected = True - self.pipe_io.send_message(b'deploy') - - def handle_pipe_event(self, message_bytes): - if not self.expected: - raise ValueError( - f'{self.pipe_io.in_pipe}: There is nothing to deploy.' - ) - - self.expected = False - if message_bytes == b'true': - self.server_connector.send_message(self.onsuccess, scope=Scope.WEBSOCKET) - elif message_bytes == b'false': - self.server_connector.send_message(self.onerror, scope=Scope.WEBSOCKET) - else: - raise ValueError( - f'{self.pipe_io.in_pipe}: Expected "true" or "false".' - ) - - -class IdePipeIOEventHandler(PipeIOEventHandlerBase): - """ - Manipulates the content of the IDE. - You can observe a file, and when the user edits it, you will receive - the new contents where newlines are escaped as "\\n". - In order to overwrite the file, send an escaped text back to the pipe. - Since the pipe doesn't know if the file is selected initially in the IDE, - you have to provide this information by yourself with @selected, - but it will track it later on. - """ - - # pylint: disable=too-many-arguments - def __init__( - self, in_pipe_path, out_pipe_path, filename, - permissions=DEFAULT_PERMISSIONS, - selected=True - ): - self.selected = selected - self.filename = filename - super().__init__('ide', in_pipe_path, out_pipe_path, permissions) - - def handle_event(self, message): - data = message['data'] - - if data['command'] == 'select': - self.selected = data['filename'] == self.filename - elif data['command'] == 'write' and self.selected: - clean = data['content'].replace('\n', '\\n') - self.pipe_io.send_message(clean.encode()) - - def handle_pipe_event(self, message_bytes): - if not self.selected: - self.server_connector.send_message({ - 'key': 'mirror', - 'data': { - 'key': 'ide', - 'data': { - 'command': 'select', - 'filename': self.filename - } - } - }) - - self.server_connector.send_message({ - 'key': 'mirror', - 'data': { - 'key': 'ide', - 'data': { - 'command': 'write', - 'content': message_bytes.decode().replace('\\n', '\n') - } - } - }) - self.server_connector.send_message({ - 'key': 'mirror', - 'data': { - 'key': 'ide', - 'data': { - 'command': 'read' - } - } - }) - - -class FSMPipeIOEventHandler(PipeIOEventHandlerBase): - """ - Handles FSM steps. - When the FSM enters the next state, you will receive a line containing - its name. To trigger a state change, send the name of the transition to - the pipe. - """ - - def __init__(self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS): - super().__init__( - ['fsm', 'fsm_update'], - in_pipe_path, out_pipe_path, permissions - ) - - def handle_event(self, message): - if 'current_state' in message['data']: - self.pipe_io.send_message(message['data']['current_state'].encode()) - - def handle_pipe_event(self, message_bytes): - self.server_connector.send_message({ - 'key': '', - 'trigger': message_bytes.decode() - }) From d9e69a6327a68e59709fe691f3a78fdbf0c58ea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 12 Jul 2019 23:26:57 +0200 Subject: [PATCH 42/68] Use new event handling model --- controller/opt/server.py | 7 ++-- solvable/src/custom_event_handlers.py | 34 +++++++++--------- solvable/src/event_handler_main.py | 50 +++++++++++++++++---------- solvable/src/pipe_io_main.py | 50 ++++----------------------- solvable/src/signal_handling.py | 11 ------ 5 files changed, 59 insertions(+), 93 deletions(-) delete mode 100644 solvable/src/signal_handling.py diff --git a/controller/opt/server.py b/controller/opt/server.py index c34f0a2..2fa5b44 100644 --- a/controller/opt/server.py +++ b/controller/opt/server.py @@ -5,11 +5,12 @@ from tornado.ioloop import IOLoop from tornado.web import RequestHandler, Application from tfw.builtins import FSMAwareEventHandler +from tfw.main import EventHandlerFactory class ControllerPostHandler(RequestHandler): - # pylint: disable=abstract-method - def initialize(self, **kwargs): # pylint: disable=arguments-differ + # pylint: disable=abstract-method,attribute-defined-outside-init,unused-argument + def initialize(self, **kwargs): self.controller = kwargs['controller'] def post(self, *args, **kwargs): @@ -26,6 +27,8 @@ class ControllerEventHandler(FSMAwareEventHandler): if __name__ == '__main__': controller = ControllerEventHandler('controller') + controller_eh = EventHandlerFactory().build(controller) + application = Application([( f'/{os.environ["SECRET"]}', ControllerPostHandler, diff --git a/solvable/src/custom_event_handlers.py b/solvable/src/custom_event_handlers.py index ca9749b..0789dba 100644 --- a/solvable/src/custom_event_handlers.py +++ b/solvable/src/custom_event_handlers.py @@ -2,25 +2,25 @@ import logging from ast import literal_eval from tfw.components import MessageSender -from tfw.builtins import TFWServerUplinkConnector -from tfw.builtins import EventHandler, FSMAwareEventHandler, TerminalCommandsEventHandler +from tfw.builtins import TerminalCommandsEventHandler +from tfw.main import TFWUplinkConnector + LOG = logging.getLogger(__name__) -class CenatorEventHandler(EventHandler): +class CenatorEventHandler: + keys = ['history.bash'] + # pylint: disable=no-self-use """ Logs commands executed in terminal to messages and invokes an additional callback function to handle special commands. !! Please remove from production code. !! """ - def __init__(self, key): - super().__init__(key) - - def handle_event(self, message): + def handle_event(self, message, server_connector): command = message['value'] LOG.debug('User executed command: "%s"', command) - MessageSender(self.server_connector).send('JOHN CENA', f'You\'ve executed "{command}"') + MessageSender(server_connector).send('JOHN CENA', f'You\'ve executed "{command}"') class TestCommandsEventHandler(TerminalCommandsEventHandler): @@ -40,7 +40,7 @@ class TestCommandsEventHandler(TerminalCommandsEventHandler): """ if not args: message_template = """'{"key": "", "data": {"command": ""}}'""" - TFWServerUplinkConnector().send_message({ + TFWUplinkConnector().send_message({ 'key': 'shell', 'data': { 'command': 'write', @@ -48,24 +48,22 @@ class TestCommandsEventHandler(TerminalCommandsEventHandler): } }) else: - TFWServerUplinkConnector().send_message(literal_eval(args[0])) + TFWUplinkConnector().send_message(literal_eval(args[0])) -class MessageFSMStepsEventHandler(FSMAwareEventHandler): +class MessageFSMStepsEventHandler: + # pylint: disable=no-self-use """ This example EventHandler is capable of detecting FSM state. !! Please remove from production code !! """ - def handle_event(self, message): - pass - - def handle_fsm_step(self, **kwargs): + def handle_event(self, message, server_connector): """ When the FSM steps this method is invoked. Receives a 'data' field from an fsm_update message as kwargs. """ - MessageSender(self.server_connector).send( + MessageSender(server_connector).send( 'FSM info', - f'FSM has stepped from state "{kwargs["last_event"]["from_state"]}" ' - f'to state "{kwargs["current_state"]}" in response to trigger "{kwargs["last_event"]["trigger"]}"' + f'FSM has stepped from state "{message["last_event"]["from_state"]}" ' + f'to state "{message["current_state"]}" in response to trigger "{message["last_event"]["trigger"]}"' ) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index b31bd2d..fc8e588 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -5,73 +5,87 @@ from functools import partial from tornado.ioloop import IOLoop from tfw.fsm import YamlFSM +from tfw.event_handlers import FSMAwareEventHandler from tfw.builtins import IdeEventHandler, TerminalEventHandler, FrontendEventHandler from tfw.builtins import LogMonitoringEventHandler, ProcessManagingEventHandler from tfw.builtins import DirectorySnapshottingEventHandler, FSMManagingEventHandler -from tfw.config import TFWENV +from tfw.main import EventHandlerFactory, setup_signal_handlers from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter +from tfw.config import TFWENV from tao.config import TAOENV from custom_event_handlers import MessageFSMStepsEventHandler from custom_event_handlers import CenatorEventHandler, TestCommandsEventHandler -from signal_handling import setup_signal_handlers + LOG = logging.getLogger(__name__) def main(): - # pylint: disable=unused-variable + # pylint: disable=unused-variable,too-many-locals Logger([ Log(stderr, LogFormatter(20)), Log(TFWENV.LOGFILE, VerboseLogFormatter()) ]).start() + eh_factory = EventHandlerFactory() # TFW component EventHandlers (builtins, required for their respective functionalities) fsm = FSMManagingEventHandler( # TFW FSM - key='fsm', fsm_type=partial( YamlFSM, 'test_fsm.yml', {} # jinja2 variables, use empty dict to enable jinja2 parsing without any variables ) ) + fsm_eh = eh_factory.build(fsm) + ide = IdeEventHandler( # Web IDE backend - key='ide', allowed_directories=[TFWENV.IDE_WD, TFWENV.WEBSERVICE_DIR], directory=TFWENV.IDE_WD, exclude=['*.pyc'] ) - terminal = TerminalEventHandler( # Web shell backend - key='shell' - ) - cenator = CenatorEventHandler('history.bash') # Reacts to terminal commands - commands = TestCommandsEventHandler( # Catches special commands - key='history.bash', - bashrc=f'/home/{TAOENV.USER}/.bashrc' - ) + ide_eh = eh_factory.build(ide) + + terminal = TerminalEventHandler() # Web shell backend + terminal_eh = eh_factory.build(terminal) + processmanager = ProcessManagingEventHandler( # Handles 'deploy' button clicks - key='processmanager', log_tail=2000 ) + processmanager_eh = eh_factory.build(processmanager) + logmonitor = LogMonitoringEventHandler( # Sends live logs of webservice process to frontend - key='logmonitor', process_name='webservice', log_tail=2000 ) + logmonitor_eh = eh_factory.build(logmonitor) + snapshot = DirectorySnapshottingEventHandler( # Manages filesystem snapshots of directories - key='snapshot', directories=[ TFWENV.IDE_WD, TFWENV.WEBSERVICE_DIR ] ) + snapshot_eh = eh_factory.build(snapshot) + frontend = FrontendEventHandler() # Proxies frontend API calls to frontend + frontend_eh = eh_factory.build(frontend) # Your custom event handlers - message_fsm_steps_eh = MessageFSMStepsEventHandler( - key='test' + cenator = CenatorEventHandler() + cenator_eh = eh_factory.build(cenator) + + message_fsm_steps = MessageFSMStepsEventHandler() + message_fsm_steps_eh = eh_factory.build( + message_fsm_steps, + event_handler_type=FSMAwareEventHandler ) + commands = TestCommandsEventHandler( # Catches special commands + bashrc=f'/home/{TAOENV.USER}/.bashrc' + ) + commands_eh = eh_factory.build(commands) + setup_signal_handlers() IOLoop.instance().start() diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index 27d886f..7a44dc3 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -6,14 +6,8 @@ from tornado.ioloop import IOLoop from tfw.builtins import PipeIOEventHandler from tfw.config import TFWENV from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter +from tfw.main import EventHandlerFactory, setup_signal_handlers -from pipe_io_auxlib import ( - SignMessagePipeIOEventHandler, VerifyMessagePipeIOEventHandler, - BotPipeIOEventHandler, - DeployPipeIOEventHandler, IdePipeIOEventHandler, - FSMPipeIOEventHandler -) -from signal_handling import setup_signal_handlers LOG = logging.getLogger(__name__) @@ -25,46 +19,14 @@ def main(): Log(TFWENV.LOGFILE, VerboseLogFormatter()) ]).start() + eh_factory = EventHandlerFactory() + json_pipe = PipeIOEventHandler( - '', - '/tmp/tfw_json_send', - '/tmp/tfw_json_recv' + '/tmp/tfw_send', + '/tmp/tfw_recv' ) + json_pipe_eh = eh_factory.build(json_pipe) - sign_pipe = SignMessagePipeIOEventHandler( - '/tmp/tfw_sign_send', - '/tmp/tfw_sign_recv', - forwarding=True - ) - - verify_pipe = VerifyMessagePipeIOEventHandler( - '/tmp/tfw_verify_send', - '/tmp/tfw_verify_recv' - ) - - bot_pipe = BotPipeIOEventHandler( - '/tmp/tfw_bot_send', - '/tmp/tfw_bot_recv', - permissions=0o666 - ) - - deploy_pipe = DeployPipeIOEventHandler( - '/tmp/tfw_deploy_send', - '/tmp/tfw_deploy_recv', - 'webservice' - ) - - ide_pipe = IdePipeIOEventHandler( - '/tmp/tfw_ide_send', - '/tmp/tfw_ide_recv', - 'user_ops.py', - selected=True - ) - - fsm_pipe = FSMPipeIOEventHandler( - '/tmp/tfw_fsm_send', - '/tmp/tfw_fsm_recv' - ) setup_signal_handlers() IOLoop.instance().start() diff --git a/solvable/src/signal_handling.py b/solvable/src/signal_handling.py deleted file mode 100644 index 21294d2..0000000 --- a/solvable/src/signal_handling.py +++ /dev/null @@ -1,11 +0,0 @@ -from signal import signal, SIGTERM, SIGINT - -from tfw.builtins import EventHandler - - -def setup_signal_handlers(): - def stop(*_): - EventHandler.stop_all_instances() - exit(0) - signal(SIGTERM, stop) - signal(SIGINT, stop) From 290c301d27897f314f466a3888bf0b25915a55a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 15 Jul 2019 13:43:34 +0200 Subject: [PATCH 43/68] Simplify *main setup --- solvable/src/custom_event_handlers.py | 30 ++---------- solvable/src/event_handler_main.py | 70 ++++++++++++--------------- solvable/src/pipe_io_main.py | 6 +-- 3 files changed, 37 insertions(+), 69 deletions(-) diff --git a/solvable/src/custom_event_handlers.py b/solvable/src/custom_event_handlers.py index 0789dba..8c2b82e 100644 --- a/solvable/src/custom_event_handlers.py +++ b/solvable/src/custom_event_handlers.py @@ -11,33 +11,16 @@ LOG = logging.getLogger(__name__) class CenatorEventHandler: keys = ['history.bash'] - # pylint: disable=no-self-use - """ - Logs commands executed in terminal to messages and invokes an - additional callback function to handle special commands. - !! Please remove from production code. !! - """ - def handle_event(self, message, server_connector): + + def handle_event(self, message, server_connector): # pylint: disable=no-self-use command = message['value'] LOG.debug('User executed command: "%s"', command) MessageSender(server_connector).send('JOHN CENA', f'You\'ve executed "{command}"') class TestCommandsEventHandler(TerminalCommandsEventHandler): - """ - Some example commands useful for debugging. - !! Please remove from production code !! and inherit your own - class from TerminalCommands if you need to define custom - commands in your challenge. - """ - # pylint: disable=unused-argument, attribute-defined-outside-init, no-self-use + # pylint: disable=unused-argument,attribute-defined-outside-init,no-self-use def command_sendmessage(self, *args): - """ - Insert TFW message template as first argument if executed without args. - - Evaluate first argumen as a dict and send it to the frontend. - This is useful for playing around with frontend APIs. - """ if not args: message_template = """'{"key": "", "data": {"command": ""}}'""" TFWUplinkConnector().send_message({ @@ -52,12 +35,7 @@ class TestCommandsEventHandler(TerminalCommandsEventHandler): class MessageFSMStepsEventHandler: - # pylint: disable=no-self-use - """ - This example EventHandler is capable of detecting FSM state. - !! Please remove from production code !! - """ - def handle_event(self, message, server_connector): + def handle_event(self, message, server_connector): # pylint: disable=no-self-use """ When the FSM steps this method is invoked. Receives a 'data' field from an fsm_update message as kwargs. diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index fc8e588..de74952 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -22,69 +22,61 @@ LOG = logging.getLogger(__name__) def main(): - # pylint: disable=unused-variable,too-many-locals + # pylint: disable=unused-variable Logger([ Log(stderr, LogFormatter(20)), Log(TFWENV.LOGFILE, VerboseLogFormatter()) ]).start() - eh_factory = EventHandlerFactory() - # TFW component EventHandlers (builtins, required for their respective functionalities) - fsm = FSMManagingEventHandler( # TFW FSM + eh_factory = EventHandlerFactory() + # TFW builtin EventHandlers (required for their respective functionalities) + # TFW FSM + fsm_eh = eh_factory.build(FSMManagingEventHandler( fsm_type=partial( YamlFSM, 'test_fsm.yml', - {} # jinja2 variables, use empty dict to enable jinja2 parsing without any variables + {} # jinja2 variables, empty dict enables jinja2 without any variables ) - ) - fsm_eh = eh_factory.build(fsm) - - ide = IdeEventHandler( # Web IDE backend + )) + # Web IDE backend + ide_eh = eh_factory.build(IdeEventHandler( allowed_directories=[TFWENV.IDE_WD, TFWENV.WEBSERVICE_DIR], directory=TFWENV.IDE_WD, exclude=['*.pyc'] - ) - ide_eh = eh_factory.build(ide) - - terminal = TerminalEventHandler() # Web shell backend - terminal_eh = eh_factory.build(terminal) - - processmanager = ProcessManagingEventHandler( # Handles 'deploy' button clicks + )) + # Web shell backend + terminal_eh = eh_factory.build(TerminalEventHandler()) + # Handles 'deploy' button clicks + processmanager_eh = eh_factory.build(ProcessManagingEventHandler( log_tail=2000 - ) - processmanager_eh = eh_factory.build(processmanager) - - logmonitor = LogMonitoringEventHandler( # Sends live logs of webservice process to frontend + )) + # Sends live logs of webservice process to frontend + logmonitor_eh = eh_factory.build(LogMonitoringEventHandler( process_name='webservice', log_tail=2000 - ) - logmonitor_eh = eh_factory.build(logmonitor) - - snapshot = DirectorySnapshottingEventHandler( # Manages filesystem snapshots of directories + )) + # Manages filesystem snapshots of directories + snapshot_eh = eh_factory.build(DirectorySnapshottingEventHandler( directories=[ TFWENV.IDE_WD, TFWENV.WEBSERVICE_DIR ] - ) - snapshot_eh = eh_factory.build(snapshot) + )) + # Proxies frontend API calls to frontend + frontend_eh = eh_factory.build(FrontendEventHandler()) - frontend = FrontendEventHandler() # Proxies frontend API calls to frontend - frontend_eh = eh_factory.build(frontend) - - # Your custom event handlers - cenator = CenatorEventHandler() - cenator_eh = eh_factory.build(cenator) - - message_fsm_steps = MessageFSMStepsEventHandler() + # Replace these with your custom event handlers + # Echoes executed commands to messages + cenator_eh = eh_factory.build(CenatorEventHandler()) + # Echoes FSM steps message_fsm_steps_eh = eh_factory.build( - message_fsm_steps, + MessageFSMStepsEventHandler(), event_handler_type=FSMAwareEventHandler ) - - commands = TestCommandsEventHandler( # Catches special commands + # Catches special commands + commands_eh = eh_factory.build(TestCommandsEventHandler( bashrc=f'/home/{TAOENV.USER}/.bashrc' - ) - commands_eh = eh_factory.build(commands) + )) setup_signal_handlers() IOLoop.instance().start() diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index 7a44dc3..89b7008 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -21,12 +21,10 @@ def main(): eh_factory = EventHandlerFactory() - json_pipe = PipeIOEventHandler( + json_pipe_eh = eh_factory.build(PipeIOEventHandler( '/tmp/tfw_send', '/tmp/tfw_recv' - ) - json_pipe_eh = eh_factory.build(json_pipe) - + )) setup_signal_handlers() IOLoop.instance().start() From 9095a9f85a13bbcb594e31773be4637eafe73f91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 24 Jul 2019 15:52:38 +0200 Subject: [PATCH 44/68] Comply new package structure --- controller/Dockerfile | 4 +- controller/opt/server.py | 23 ++++----- ...m_event_handlers.py => custom_handlers.py} | 29 ++++++------ solvable/src/event_handler_main.py | 47 +++++++++++-------- solvable/src/pipe_io_main.py | 4 +- solvable/src/test_fsm.py | 6 +-- 6 files changed, 58 insertions(+), 55 deletions(-) rename solvable/src/{custom_event_handlers.py => custom_handlers.py} (57%) diff --git a/controller/Dockerfile b/controller/Dockerfile index d09b2fd..6c1afeb 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -2,8 +2,8 @@ FROM avatao/controller:debian-buster USER root ENV PYTHONPATH="/usr/local/lib" \ - TFW_PUBLISHER_PORT=7654 \ - TFW_RECEIVER_PORT=8765 \ + TFW_PUB_PORT=7654 \ + TFW_PULL_PORT=8765 \ TFW_AUTH_KEY="/tmp/tfw-auth.key" \ CONTROLLER_PORT=5555 diff --git a/controller/opt/server.py b/controller/opt/server.py index 2fa5b44..f2c1bb6 100644 --- a/controller/opt/server.py +++ b/controller/opt/server.py @@ -4,8 +4,8 @@ import json from tornado.ioloop import IOLoop from tornado.web import RequestHandler, Application -from tfw.builtins import FSMAwareEventHandler -from tfw.main import EventHandlerFactory +from tfw.event_handlers import FSMAwareEventHandler +from tfw.main import EventHandlerFactory, setup_signal_handlers class ControllerPostHandler(RequestHandler): @@ -20,23 +20,18 @@ class ControllerPostHandler(RequestHandler): })) -class ControllerEventHandler(FSMAwareEventHandler): - def handle_event(self, message): - pass - - if __name__ == '__main__': - controller = ControllerEventHandler('controller') - controller_eh = EventHandlerFactory().build(controller) + controller_eh = EventHandlerFactory().build( + lambda *_: None, + event_handler_type=FSMAwareEventHandler + ) application = Application([( f'/{os.environ["SECRET"]}', ControllerPostHandler, - {'controller': controller} + {'controller': controller_eh} )]) application.listen(os.environ['CONTROLLER_PORT']) - try: - IOLoop.instance().start() - finally: - controller.cleanup() + setup_signal_handlers() + IOLoop.instance().start() diff --git a/solvable/src/custom_event_handlers.py b/solvable/src/custom_handlers.py similarity index 57% rename from solvable/src/custom_event_handlers.py rename to solvable/src/custom_handlers.py index 8c2b82e..48c7220 100644 --- a/solvable/src/custom_event_handlers.py +++ b/solvable/src/custom_handlers.py @@ -1,15 +1,15 @@ import logging from ast import literal_eval -from tfw.components import MessageSender -from tfw.builtins import TerminalCommandsEventHandler +from tfw.components.frontend import MessageSender +from tfw.components.terminal import TerminalCommandsHandler from tfw.main import TFWUplinkConnector LOG = logging.getLogger(__name__) -class CenatorEventHandler: +class CenatorHandler: keys = ['history.bash'] def handle_event(self, message, server_connector): # pylint: disable=no-self-use @@ -18,7 +18,7 @@ class CenatorEventHandler: MessageSender(server_connector).send('JOHN CENA', f'You\'ve executed "{command}"') -class TestCommandsEventHandler(TerminalCommandsEventHandler): +class TestCommandsHandler(TerminalCommandsHandler): # pylint: disable=unused-argument,attribute-defined-outside-init,no-self-use def command_sendmessage(self, *args): if not args: @@ -34,14 +34,13 @@ class TestCommandsEventHandler(TerminalCommandsEventHandler): TFWUplinkConnector().send_message(literal_eval(args[0])) -class MessageFSMStepsEventHandler: - def handle_event(self, message, server_connector): # pylint: disable=no-self-use - """ - When the FSM steps this method is invoked. - Receives a 'data' field from an fsm_update message as kwargs. - """ - MessageSender(server_connector).send( - 'FSM info', - f'FSM has stepped from state "{message["last_event"]["from_state"]}" ' - f'to state "{message["current_state"]}" in response to trigger "{message["last_event"]["trigger"]}"' - ) +def messageFSMStepsHandler(message, server_connector): + """ + When the FSM steps this method is invoked. + Receives a 'data' field from an fsm_update message as kwargs. + """ + MessageSender(server_connector).send( + 'FSM info', + f'FSM has stepped from state "{message["last_event"]["from_state"]}" ' + f'to state "{message["current_state"]}" in response to trigger "{message["last_event"]["trigger"]}"' + ) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index de74952..fe670e5 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -6,16 +6,17 @@ from tornado.ioloop import IOLoop from tfw.fsm import YamlFSM from tfw.event_handlers import FSMAwareEventHandler -from tfw.builtins import IdeEventHandler, TerminalEventHandler, FrontendEventHandler -from tfw.builtins import LogMonitoringEventHandler, ProcessManagingEventHandler -from tfw.builtins import DirectorySnapshottingEventHandler, FSMManagingEventHandler +from tfw.components.ide import IdeHandler +from tfw.components.terminal import TerminalHandler +from tfw.components.frontend import FrontendHandler +from tfw.components.process_management import ProcessHandler, ProcessLogHandler +from tfw.components.snapshots import SnapshotHandler +from tfw.components.fsm import FSMHandler from tfw.main import EventHandlerFactory, setup_signal_handlers from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter -from tfw.config import TFWENV -from tao.config import TAOENV +from tfw.config import TFWENV, TAOENV -from custom_event_handlers import MessageFSMStepsEventHandler -from custom_event_handlers import CenatorEventHandler, TestCommandsEventHandler +from custom_handlers import CenatorHandler, TestCommandsHandler, messageFSMStepsHandler LOG = logging.getLogger(__name__) @@ -31,7 +32,7 @@ def main(): eh_factory = EventHandlerFactory() # TFW builtin EventHandlers (required for their respective functionalities) # TFW FSM - fsm_eh = eh_factory.build(FSMManagingEventHandler( + fsm_eh = eh_factory.build(FSMHandler( fsm_type=partial( YamlFSM, 'test_fsm.yml', @@ -39,42 +40,50 @@ def main(): ) )) # Web IDE backend - ide_eh = eh_factory.build(IdeEventHandler( + ide_eh = eh_factory.build(IdeHandler( allowed_directories=[TFWENV.IDE_WD, TFWENV.WEBSERVICE_DIR], directory=TFWENV.IDE_WD, exclude=['*.pyc'] )) # Web shell backend - terminal_eh = eh_factory.build(TerminalEventHandler()) + terminal_eh = eh_factory.build(TerminalHandler( + port=TFWENV.TERMINADO_PORT, + user=TAOENV.USER, + workind_directory=TFWENV.TERMINADO_WD, + histfile=TFWENV.HISTFILE + )) # Handles 'deploy' button clicks - processmanager_eh = eh_factory.build(ProcessManagingEventHandler( - log_tail=2000 + processmanager_eh = eh_factory.build(ProcessHandler( + supervisor_uri=TFWENV.SUPERVISOR_HTTP_URI, + log_tail=2000, )) # Sends live logs of webservice process to frontend - logmonitor_eh = eh_factory.build(LogMonitoringEventHandler( + logmonitor_eh = eh_factory.build(ProcessLogHandler( process_name='webservice', + supervisor_uri=TFWENV.SUPERVISOR_HTTP_URI, log_tail=2000 )) # Manages filesystem snapshots of directories - snapshot_eh = eh_factory.build(DirectorySnapshottingEventHandler( + snapshot_eh = eh_factory.build(SnapshotHandler( directories=[ TFWENV.IDE_WD, TFWENV.WEBSERVICE_DIR - ] + ], + snapshots_dir=TFWENV.SNAPSHOTS_DIR )) # Proxies frontend API calls to frontend - frontend_eh = eh_factory.build(FrontendEventHandler()) + frontend_eh = eh_factory.build(FrontendHandler()) # Replace these with your custom event handlers # Echoes executed commands to messages - cenator_eh = eh_factory.build(CenatorEventHandler()) + cenator_eh = eh_factory.build(CenatorHandler()) # Echoes FSM steps message_fsm_steps_eh = eh_factory.build( - MessageFSMStepsEventHandler(), + messageFSMStepsHandler, event_handler_type=FSMAwareEventHandler ) # Catches special commands - commands_eh = eh_factory.build(TestCommandsEventHandler( + commands_eh = eh_factory.build(TestCommandsHandler( bashrc=f'/home/{TAOENV.USER}/.bashrc' )) diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index 89b7008..1b6abf9 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -3,7 +3,7 @@ from sys import stderr from tornado.ioloop import IOLoop -from tfw.builtins import PipeIOEventHandler +from tfw.components.pipe_io import PipeIOHandler from tfw.config import TFWENV from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter from tfw.main import EventHandlerFactory, setup_signal_handlers @@ -21,7 +21,7 @@ def main(): eh_factory = EventHandlerFactory() - json_pipe_eh = eh_factory.build(PipeIOEventHandler( + json_pipe_eh = eh_factory.build(PipeIOHandler( '/tmp/tfw_send', '/tmp/tfw_recv' )) diff --git a/solvable/src/test_fsm.py b/solvable/src/test_fsm.py index f4f7639..0d41590 100644 --- a/solvable/src/test_fsm.py +++ b/solvable/src/test_fsm.py @@ -3,8 +3,8 @@ from os.path import exists from tfw.fsm import LinearFSM -from tfw.components import MessageSender -from tfw.builtins import TFWServerUplinkConnector +from tfw.components.frontend import MessageSender +from tfw.main import TFWUplinkConnector class TestFSM(LinearFSM): @@ -12,7 +12,7 @@ class TestFSM(LinearFSM): def __init__(self): super().__init__(6) - self.uplink = TFWServerUplinkConnector() + self.uplink = TFWUplinkConnector() self.message_sender = MessageSender(self.uplink) self.subscribe_predicate('step_3', self.step_3_allowed) From 27cc80d36c65ad2551dd4e8c28b5469443a9f104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 24 Jul 2019 16:33:21 +0200 Subject: [PATCH 45/68] Remove unnecessary newlines --- solvable/src/custom_handlers.py | 1 - solvable/src/event_handler_main.py | 1 - solvable/src/pipe_io_main.py | 1 - 3 files changed, 3 deletions(-) diff --git a/solvable/src/custom_handlers.py b/solvable/src/custom_handlers.py index 48c7220..96f4597 100644 --- a/solvable/src/custom_handlers.py +++ b/solvable/src/custom_handlers.py @@ -5,7 +5,6 @@ from tfw.components.frontend import MessageSender from tfw.components.terminal import TerminalCommandsHandler from tfw.main import TFWUplinkConnector - LOG = logging.getLogger(__name__) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index fe670e5..56bc57d 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -18,7 +18,6 @@ from tfw.config import TFWENV, TAOENV from custom_handlers import CenatorHandler, TestCommandsHandler, messageFSMStepsHandler - LOG = logging.getLogger(__name__) diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index 1b6abf9..2aab217 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -8,7 +8,6 @@ from tfw.config import TFWENV from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter from tfw.main import EventHandlerFactory, setup_signal_handlers - LOG = logging.getLogger(__name__) From b032da898305e801900eb78177e408e1b3beb6bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sun, 28 Jul 2019 21:04:22 +0200 Subject: [PATCH 46/68] Use IOLoop.current() instead of deprecated instance() --- controller/opt/server.py | 2 +- solvable/src/event_handler_main.py | 2 +- solvable/src/pipe_io_main.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/controller/opt/server.py b/controller/opt/server.py index f2c1bb6..712c1df 100644 --- a/controller/opt/server.py +++ b/controller/opt/server.py @@ -34,4 +34,4 @@ if __name__ == '__main__': application.listen(os.environ['CONTROLLER_PORT']) setup_signal_handlers() - IOLoop.instance().start() + IOLoop.current().start() diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 56bc57d..bc44e16 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -87,7 +87,7 @@ def main(): )) setup_signal_handlers() - IOLoop.instance().start() + IOLoop.current().start() if __name__ == '__main__': diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index 2aab217..f833418 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -26,7 +26,7 @@ def main(): )) setup_signal_handlers() - IOLoop.instance().start() + IOLoop.current().start() if __name__ == '__main__': From 01b247e49474a9a01d185f4f3a86a185908b7a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 30 Jul 2019 15:17:47 +0200 Subject: [PATCH 47/68] Follow API changes --- solvable/src/custom_handlers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/solvable/src/custom_handlers.py b/solvable/src/custom_handlers.py index 96f4597..67e48d2 100644 --- a/solvable/src/custom_handlers.py +++ b/solvable/src/custom_handlers.py @@ -11,10 +11,10 @@ LOG = logging.getLogger(__name__) class CenatorHandler: keys = ['history.bash'] - def handle_event(self, message, server_connector): # pylint: disable=no-self-use + def handle_event(self, message, connector): # pylint: disable=no-self-use command = message['value'] LOG.debug('User executed command: "%s"', command) - MessageSender(server_connector).send('JOHN CENA', f'You\'ve executed "{command}"') + MessageSender(connector).send('JOHN CENA', f'You\'ve executed "{command}"') class TestCommandsHandler(TerminalCommandsHandler): @@ -33,12 +33,12 @@ class TestCommandsHandler(TerminalCommandsHandler): TFWUplinkConnector().send_message(literal_eval(args[0])) -def messageFSMStepsHandler(message, server_connector): +def messageFSMStepsHandler(message, connector): """ When the FSM steps this method is invoked. Receives a 'data' field from an fsm_update message as kwargs. """ - MessageSender(server_connector).send( + MessageSender(connector).send( 'FSM info', f'FSM has stepped from state "{message["last_event"]["from_state"]}" ' f'to state "{message["current_state"]}" in response to trigger "{message["last_event"]["trigger"]}"' From a1ad0e6356cf0a00a2d9ce7052c579f5d6545af2 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 1 Aug 2019 16:06:10 +0200 Subject: [PATCH 48/68] Fix HOTRELOAD mount path --- hack/libhack/challenge.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/libhack/challenge.sh b/hack/libhack/challenge.sh index 4ed1377..2add273 100644 --- a/hack/libhack/challenge.sh +++ b/hack/libhack/challenge.sh @@ -38,7 +38,7 @@ challenge::run() { local mount_volumes if [[ "${HOTRELOAD:-0}" == "1" ]]; then if [[ -d "${BASEIMAGE_PATH}" ]]; then - mount_baseimage="-v ${BASEIMAGE_PATH}/lib/tfw:/usr/local/lib/tfw" + mount_baseimage="-v ${BASEIMAGE_PATH}/tfw:/usr/local/lib/tfw" fi mount_challenge="-v ${CHALLENGE_PATH}/solvable/src:/.tfw/builtin_event_handlers" mount_volumes="${mount_baseimage:-} ${mount_challenge}" From e366a2652ba35a0f838b4e4327dbe4119179c706 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 7 Aug 2019 10:02:16 +0200 Subject: [PATCH 49/68] Simplify MessageSender, IDE and terminal handler --- solvable/Dockerfile | 1 - solvable/src/custom_handlers.py | 6 +++--- solvable/src/event_handler_main.py | 20 +++++++++----------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/solvable/Dockerfile b/solvable/Dockerfile index d61c83a..2e82771 100644 --- a/solvable/Dockerfile +++ b/solvable/Dockerfile @@ -15,7 +15,6 @@ ADD solvable/src/webservice/frontend-deps.tar ${TFW_WEBSERVICE_DIR}/static # Create IDE directory, symlink server source and give proper permissions to AVATAO_USER RUN mkdir -p ${TFW_IDE_WD} &&\ - ln -s ${TFW_WEBSERVICE_DIR}/user_ops.py ${TFW_IDE_WD} &&\ chown -R ${AVATAO_USER}: "${TFW_IDE_WD}" "${TFW_WEBSERVICE_DIR}" &&\ chmod -R 755 "${TFW_IDE_WD}" "${TFW_WEBSERVICE_DIR}" diff --git a/solvable/src/custom_handlers.py b/solvable/src/custom_handlers.py index 67e48d2..bf34752 100644 --- a/solvable/src/custom_handlers.py +++ b/solvable/src/custom_handlers.py @@ -14,7 +14,7 @@ class CenatorHandler: def handle_event(self, message, connector): # pylint: disable=no-self-use command = message['value'] LOG.debug('User executed command: "%s"', command) - MessageSender(connector).send('JOHN CENA', f'You\'ve executed "{command}"') + MessageSender(connector).send(f'You\'ve executed "{command}"', originator='JOHN CENA') class TestCommandsHandler(TerminalCommandsHandler): @@ -39,7 +39,7 @@ def messageFSMStepsHandler(message, connector): Receives a 'data' field from an fsm_update message as kwargs. """ MessageSender(connector).send( - 'FSM info', f'FSM has stepped from state "{message["last_event"]["from_state"]}" ' - f'to state "{message["current_state"]}" in response to trigger "{message["last_event"]["trigger"]}"' + f'to state "{message["current_state"]}" in response to trigger "{message["last_event"]["trigger"]}"', + originator='FSM info' ) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index bc44e16..6c00040 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -40,15 +40,13 @@ def main(): )) # Web IDE backend ide_eh = eh_factory.build(IdeHandler( - allowed_directories=[TFWENV.IDE_WD, TFWENV.WEBSERVICE_DIR], - directory=TFWENV.IDE_WD, - exclude=['*.pyc'] + patterns=['/home/user/workdir/*', '/srv/webservice/user_ops.py'] )) # Web shell backend terminal_eh = eh_factory.build(TerminalHandler( port=TFWENV.TERMINADO_PORT, user=TAOENV.USER, - workind_directory=TFWENV.TERMINADO_WD, + working_directory=TFWENV.TERMINADO_WD, histfile=TFWENV.HISTFILE )) # Handles 'deploy' button clicks @@ -63,13 +61,13 @@ def main(): log_tail=2000 )) # Manages filesystem snapshots of directories - snapshot_eh = eh_factory.build(SnapshotHandler( - directories=[ - TFWENV.IDE_WD, - TFWENV.WEBSERVICE_DIR - ], - snapshots_dir=TFWENV.SNAPSHOTS_DIR - )) + #snapshot_eh = eh_factory.build(SnapshotHandler( + # directories=[ + # TFWENV.IDE_WD, + # TFWENV.WEBSERVICE_DIR + # ], + # snapshots_dir=TFWENV.SNAPSHOTS_DIR + #)) # Proxies frontend API calls to frontend frontend_eh = eh_factory.build(FrontendHandler()) From 4c163e1085dfa9b6778083e5ba74cefc09355c4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 8 Aug 2019 15:10:24 +0200 Subject: [PATCH 50/68] Remove snaphotting from default event_handler_main --- solvable/src/event_handler_main.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 6c00040..adb3b90 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -60,14 +60,6 @@ def main(): supervisor_uri=TFWENV.SUPERVISOR_HTTP_URI, log_tail=2000 )) - # Manages filesystem snapshots of directories - #snapshot_eh = eh_factory.build(SnapshotHandler( - # directories=[ - # TFWENV.IDE_WD, - # TFWENV.WEBSERVICE_DIR - # ], - # snapshots_dir=TFWENV.SNAPSHOTS_DIR - #)) # Proxies frontend API calls to frontend frontend_eh = eh_factory.build(FrontendHandler()) From f4840f4a6b421d9e347a0bf810d4ed3dc881bd7c Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 14 Aug 2019 14:14:57 +0200 Subject: [PATCH 51/68] Create event handler for ProxyPipeConnector --- solvable/src/pipe_io_main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index f833418..8f10cad 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -4,6 +4,7 @@ from sys import stderr from tornado.ioloop import IOLoop from tfw.components.pipe_io import PipeIOHandler +from tfw.components.pipe_connector import ProxyPipeConnectorHandler from tfw.config import TFWENV from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter from tfw.main import EventHandlerFactory, setup_signal_handlers @@ -24,6 +25,7 @@ def main(): '/tmp/tfw_send', '/tmp/tfw_recv' )) + proxy_pipe_eh = eh_factory.build(ProxyPipeConnectorHandler('/tmp/pipes')) setup_signal_handlers() IOLoop.current().start() From 32bab90ac79a178c8b96c64189e3d829477ec885 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 15 Aug 2019 14:13:56 +0200 Subject: [PATCH 52/68] Fix import path --- solvable/src/pipe_io_main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index 8f10cad..c49c183 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -3,8 +3,7 @@ from sys import stderr from tornado.ioloop import IOLoop -from tfw.components.pipe_io import PipeIOHandler -from tfw.components.pipe_connector import ProxyPipeConnectorHandler +from tfw.components.pipe_io import PipeIOHandler, ProxyPipeConnectorHandler from tfw.config import TFWENV from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter from tfw.main import EventHandlerFactory, setup_signal_handlers From 419a6945971f745abfab223fce43a04c4bf4db68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 15 Aug 2019 14:21:41 +0200 Subject: [PATCH 53/68] Remove unused handler import --- solvable/src/event_handler_main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index adb3b90..310bbdb 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -10,7 +10,6 @@ from tfw.components.ide import IdeHandler from tfw.components.terminal import TerminalHandler from tfw.components.frontend import FrontendHandler from tfw.components.process_management import ProcessHandler, ProcessLogHandler -from tfw.components.snapshots import SnapshotHandler from tfw.components.fsm import FSMHandler from tfw.main import EventHandlerFactory, setup_signal_handlers from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter From 4bd8ee1cf2ebea80fb79c1d51cb1db1483968eb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 15 Aug 2019 14:37:32 +0200 Subject: [PATCH 54/68] Use new ConsoleLogsHandler to write process logs to frontend --- solvable/src/event_handler_main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 310bbdb..5dd0596 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -8,7 +8,7 @@ from tfw.fsm import YamlFSM from tfw.event_handlers import FSMAwareEventHandler from tfw.components.ide import IdeHandler from tfw.components.terminal import TerminalHandler -from tfw.components.frontend import FrontendHandler +from tfw.components.frontend import FrontendProxyHandler, ConsoleLogsHandler from tfw.components.process_management import ProcessHandler, ProcessLogHandler from tfw.components.fsm import FSMHandler from tfw.main import EventHandlerFactory, setup_signal_handlers @@ -50,8 +50,7 @@ def main(): )) # Handles 'deploy' button clicks processmanager_eh = eh_factory.build(ProcessHandler( - supervisor_uri=TFWENV.SUPERVISOR_HTTP_URI, - log_tail=2000, + supervisor_uri=TFWENV.SUPERVISOR_HTTP_URI )) # Sends live logs of webservice process to frontend logmonitor_eh = eh_factory.build(ProcessLogHandler( @@ -60,7 +59,9 @@ def main(): log_tail=2000 )) # Proxies frontend API calls to frontend - frontend_eh = eh_factory.build(FrontendHandler()) + frontend_eh = eh_factory.build(FrontendProxyHandler()) + # Writes live logs to console on frontend + console_logs_eh = eh_factory.build(ConsoleLogsHandler(stream='stdout')) # Replace these with your custom event handlers # Echoes executed commands to messages From 11b3ac8eef23f064e7afe4962ba9b57619e490de Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 15 Aug 2019 15:40:09 +0200 Subject: [PATCH 55/68] Move initial FSM trigger to backend --- solvable/src/event_handler_main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 5dd0596..cde0168 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -35,7 +35,8 @@ def main(): YamlFSM, 'test_fsm.yml', {} # jinja2 variables, empty dict enables jinja2 without any variables - ) + ), + initial_trigger='step_1' )) # Web IDE backend ide_eh = eh_factory.build(IdeHandler( From 9fd352091014a0ae506c564cc55e9d0a571222ee Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Thu, 15 Aug 2019 17:04:27 +0200 Subject: [PATCH 56/68] Modify resource's directory structure --- hack/libhack/challenge.sh | 2 +- solvable/src/event_handler_main.py | 9 ++------- solvable/src/pipe_io_main.py | 13 ++++--------- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/hack/libhack/challenge.sh b/hack/libhack/challenge.sh index 2add273..4d4cd95 100644 --- a/hack/libhack/challenge.sh +++ b/hack/libhack/challenge.sh @@ -38,7 +38,7 @@ challenge::run() { local mount_volumes if [[ "${HOTRELOAD:-0}" == "1" ]]; then if [[ -d "${BASEIMAGE_PATH}" ]]; then - mount_baseimage="-v ${BASEIMAGE_PATH}/tfw:/usr/local/lib/tfw" + mount_baseimage="-e HOTRELOAD=1 -v ${BASEIMAGE_PATH}/tfw:/usr/local/lib/tfw" fi mount_challenge="-v ${CHALLENGE_PATH}/solvable/src:/.tfw/builtin_event_handlers" mount_volumes="${mount_baseimage:-} ${mount_challenge}" diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index cde0168..8119f08 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -1,5 +1,4 @@ import logging -from sys import stderr from functools import partial from tornado.ioloop import IOLoop @@ -11,8 +10,7 @@ from tfw.components.terminal import TerminalHandler from tfw.components.frontend import FrontendProxyHandler, ConsoleLogsHandler from tfw.components.process_management import ProcessHandler, ProcessLogHandler from tfw.components.fsm import FSMHandler -from tfw.main import EventHandlerFactory, setup_signal_handlers -from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter +from tfw.main import EventHandlerFactory, setup_logger, setup_signal_handlers from tfw.config import TFWENV, TAOENV from custom_handlers import CenatorHandler, TestCommandsHandler, messageFSMStepsHandler @@ -22,10 +20,7 @@ LOG = logging.getLogger(__name__) def main(): # pylint: disable=unused-variable - Logger([ - Log(stderr, LogFormatter(20)), - Log(TFWENV.LOGFILE, VerboseLogFormatter()) - ]).start() + setup_logger(__file__) eh_factory = EventHandlerFactory() # TFW builtin EventHandlers (required for their respective functionalities) diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index c49c183..c240130 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -1,22 +1,17 @@ import logging -from sys import stderr from tornado.ioloop import IOLoop -from tfw.components.pipe_io import PipeIOHandler, ProxyPipeConnectorHandler from tfw.config import TFWENV -from tfw.logging import Log, Logger, LogFormatter, VerboseLogFormatter -from tfw.main import EventHandlerFactory, setup_signal_handlers +from tfw.components.pipe_io import PipeIOHandler, ProxyPipeConnectorHandler +from tfw.main import EventHandlerFactory, setup_logger, setup_signal_handlers LOG = logging.getLogger(__name__) def main(): # pylint: disable=unused-variable - Logger([ - Log(stderr, LogFormatter(20)), - Log(TFWENV.LOGFILE, VerboseLogFormatter()) - ]).start() + setup_logger(__file__) eh_factory = EventHandlerFactory() @@ -24,7 +19,7 @@ def main(): '/tmp/tfw_send', '/tmp/tfw_recv' )) - proxy_pipe_eh = eh_factory.build(ProxyPipeConnectorHandler('/tmp/pipes')) + proxy_pipe_eh = eh_factory.build(ProxyPipeConnectorHandler(TFWENV.PIPES_DIR)) setup_signal_handlers() IOLoop.current().start() From c972026385ba2d98df4548f24608b0ca62631505 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 23 Aug 2019 15:33:50 +0200 Subject: [PATCH 57/68] Append --init flag to Docker run in order to get zombie processes reaped --- hack/libhack/challenge.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/libhack/challenge.sh b/hack/libhack/challenge.sh index 4d4cd95..0c93027 100644 --- a/hack/libhack/challenge.sh +++ b/hack/libhack/challenge.sh @@ -44,7 +44,8 @@ challenge::run() { mount_volumes="${mount_baseimage:-} ${mount_challenge}" fi popd - docker run --rm \ + docker run --init \ + --rm \ -p ${CHALLENGE_PORT}:${CHALLENGE_PORT} \ ${mount_volumes:-} \ ${@:-} \ From 88e1818c8656e8a93e5cb9994c0165cfa0762251 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 23 Aug 2019 15:56:03 +0200 Subject: [PATCH 58/68] Introduce intent for TFW messages --- solvable/src/event_handler_main.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 8119f08..4b1bed4 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -4,7 +4,7 @@ from functools import partial from tornado.ioloop import IOLoop from tfw.fsm import YamlFSM -from tfw.event_handlers import FSMAwareEventHandler +from tfw.event_handlers import FSMAwareEventHandler, ControlEventHandler from tfw.components.ide import IdeHandler from tfw.components.terminal import TerminalHandler from tfw.components.frontend import FrontendProxyHandler, ConsoleLogsHandler @@ -12,6 +12,7 @@ from tfw.components.process_management import ProcessHandler, ProcessLogHandler from tfw.components.fsm import FSMHandler from tfw.main import EventHandlerFactory, setup_logger, setup_signal_handlers from tfw.config import TFWENV, TAOENV +from tfw.internals.networking import Intent from custom_handlers import CenatorHandler, TestCommandsHandler, messageFSMStepsHandler @@ -32,36 +33,36 @@ def main(): {} # jinja2 variables, empty dict enables jinja2 without any variables ), initial_trigger='step_1' - )) + ), event_handler_type=ControlEventHandler) # Web IDE backend ide_eh = eh_factory.build(IdeHandler( patterns=['/home/user/workdir/*', '/srv/webservice/user_ops.py'] - )) + ), event_handler_type=ControlEventHandler) # Web shell backend terminal_eh = eh_factory.build(TerminalHandler( port=TFWENV.TERMINADO_PORT, user=TAOENV.USER, working_directory=TFWENV.TERMINADO_WD, histfile=TFWENV.HISTFILE - )) + ), event_handler_type=ControlEventHandler) # Handles 'deploy' button clicks processmanager_eh = eh_factory.build(ProcessHandler( supervisor_uri=TFWENV.SUPERVISOR_HTTP_URI - )) + ), event_handler_type=ControlEventHandler) # Sends live logs of webservice process to frontend logmonitor_eh = eh_factory.build(ProcessLogHandler( process_name='webservice', supervisor_uri=TFWENV.SUPERVISOR_HTTP_URI, log_tail=2000 - )) + ), event_handler_type=ControlEventHandler) # Proxies frontend API calls to frontend - frontend_eh = eh_factory.build(FrontendProxyHandler()) + frontend_eh = eh_factory.build(FrontendProxyHandler(), event_handler_type=ControlEventHandler) # Writes live logs to console on frontend - console_logs_eh = eh_factory.build(ConsoleLogsHandler(stream='stdout')) + console_logs_eh = eh_factory.build(ConsoleLogsHandler(stream='stdout'), event_handler_type=ControlEventHandler) # Replace these with your custom event handlers # Echoes executed commands to messages - cenator_eh = eh_factory.build(CenatorHandler()) + cenator_eh = eh_factory.build(CenatorHandler(), event_handler_type=ControlEventHandler) # Echoes FSM steps message_fsm_steps_eh = eh_factory.build( messageFSMStepsHandler, @@ -70,7 +71,7 @@ def main(): # Catches special commands commands_eh = eh_factory.build(TestCommandsHandler( bashrc=f'/home/{TAOENV.USER}/.bashrc' - )) + ), event_handler_type=ControlEventHandler) setup_signal_handlers() IOLoop.current().start() From 9404c86de7d8ddb5a4caa16abd110938082a1299 Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 23 Aug 2019 16:18:58 +0200 Subject: [PATCH 59/68] Move message queue logic to backend --- solvable/src/event_handler_main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 4b1bed4..363fae0 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -7,7 +7,7 @@ from tfw.fsm import YamlFSM from tfw.event_handlers import FSMAwareEventHandler, ControlEventHandler from tfw.components.ide import IdeHandler from tfw.components.terminal import TerminalHandler -from tfw.components.frontend import FrontendProxyHandler, ConsoleLogsHandler +from tfw.components.frontend import FrontendProxyHandler, ConsoleLogsHandler, MessageQueueHandler from tfw.components.process_management import ProcessHandler, ProcessLogHandler from tfw.components.fsm import FSMHandler from tfw.main import EventHandlerFactory, setup_logger, setup_signal_handlers @@ -57,6 +57,7 @@ def main(): ), event_handler_type=ControlEventHandler) # Proxies frontend API calls to frontend frontend_eh = eh_factory.build(FrontendProxyHandler(), event_handler_type=ControlEventHandler) + message_queue_eh = eh_factory.build(MessageQueueHandler(25), event_handler_type=ControlEventHandler) # Writes live logs to console on frontend console_logs_eh = eh_factory.build(ConsoleLogsHandler(stream='stdout'), event_handler_type=ControlEventHandler) From 620f6aa6430d7ab670a75f28fa86dcdd97aed61a Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Fri, 23 Aug 2019 16:23:21 +0200 Subject: [PATCH 60/68] Create handler for deploying in the IDE --- solvable/src/event_handler_main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 363fae0..53acd3c 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -5,14 +5,13 @@ from tornado.ioloop import IOLoop from tfw.fsm import YamlFSM from tfw.event_handlers import FSMAwareEventHandler, ControlEventHandler -from tfw.components.ide import IdeHandler +from tfw.components.ide import IdeHandler, DeployHandler from tfw.components.terminal import TerminalHandler from tfw.components.frontend import FrontendProxyHandler, ConsoleLogsHandler, MessageQueueHandler from tfw.components.process_management import ProcessHandler, ProcessLogHandler from tfw.components.fsm import FSMHandler from tfw.main import EventHandlerFactory, setup_logger, setup_signal_handlers from tfw.config import TFWENV, TAOENV -from tfw.internals.networking import Intent from custom_handlers import CenatorHandler, TestCommandsHandler, messageFSMStepsHandler @@ -38,6 +37,7 @@ def main(): ide_eh = eh_factory.build(IdeHandler( patterns=['/home/user/workdir/*', '/srv/webservice/user_ops.py'] ), event_handler_type=ControlEventHandler) + deploy_eh = eh_factory.build(DeployHandler(), event_handler_type=ControlEventHandler) # Web shell backend terminal_eh = eh_factory.build(TerminalHandler( port=TFWENV.TERMINADO_PORT, From 054d52b5fa05cac2d1d48abbe325ccde2e17920a Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Mon, 26 Aug 2019 14:46:21 +0200 Subject: [PATCH 61/68] Fix event handler types and sendmessage command --- solvable/src/custom_handlers.py | 16 ++++++++-------- solvable/src/event_handler_main.py | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/solvable/src/custom_handlers.py b/solvable/src/custom_handlers.py index bf34752..1d022fb 100644 --- a/solvable/src/custom_handlers.py +++ b/solvable/src/custom_handlers.py @@ -1,5 +1,5 @@ import logging -from ast import literal_eval +from json import loads, JSONDecodeError from tfw.components.frontend import MessageSender from tfw.components.terminal import TerminalCommandsHandler @@ -21,16 +21,16 @@ class TestCommandsHandler(TerminalCommandsHandler): # pylint: disable=unused-argument,attribute-defined-outside-init,no-self-use def command_sendmessage(self, *args): if not args: - message_template = """'{"key": "", "data": {"command": ""}}'""" + message_template = """'{"key": ""}'""" TFWUplinkConnector().send_message({ - 'key': 'shell', - 'data': { - 'command': 'write', - 'value': f'sendmessage {message_template}' - } + 'key': 'terminal.write', + 'command': f'sendmessage {message_template}' }) else: - TFWUplinkConnector().send_message(literal_eval(args[0])) + try: + TFWUplinkConnector().send_message(loads(args[0])) + except JSONDecodeError: + LOG.error('IGNORING MESSAGE: Invalid message received: %s', args[0]) def messageFSMStepsHandler(message, connector): diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 53acd3c..0a4a0d0 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -37,7 +37,7 @@ def main(): ide_eh = eh_factory.build(IdeHandler( patterns=['/home/user/workdir/*', '/srv/webservice/user_ops.py'] ), event_handler_type=ControlEventHandler) - deploy_eh = eh_factory.build(DeployHandler(), event_handler_type=ControlEventHandler) + deploy_eh = eh_factory.build(DeployHandler()) # Web shell backend terminal_eh = eh_factory.build(TerminalHandler( port=TFWENV.TERMINADO_PORT, @@ -59,11 +59,11 @@ def main(): frontend_eh = eh_factory.build(FrontendProxyHandler(), event_handler_type=ControlEventHandler) message_queue_eh = eh_factory.build(MessageQueueHandler(25), event_handler_type=ControlEventHandler) # Writes live logs to console on frontend - console_logs_eh = eh_factory.build(ConsoleLogsHandler(stream='stdout'), event_handler_type=ControlEventHandler) + console_logs_eh = eh_factory.build(ConsoleLogsHandler(stream='stdout')) # Replace these with your custom event handlers # Echoes executed commands to messages - cenator_eh = eh_factory.build(CenatorHandler(), event_handler_type=ControlEventHandler) + cenator_eh = eh_factory.build(CenatorHandler()) # Echoes FSM steps message_fsm_steps_eh = eh_factory.build( messageFSMStepsHandler, @@ -72,7 +72,7 @@ def main(): # Catches special commands commands_eh = eh_factory.build(TestCommandsHandler( bashrc=f'/home/{TAOENV.USER}/.bashrc' - ), event_handler_type=ControlEventHandler) + )) setup_signal_handlers() IOLoop.current().start() From 106b932c6b63cb8dc75fc811c33e28c40470971d Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 27 Aug 2019 13:05:52 +0200 Subject: [PATCH 62/68] Revert "Append --init flag to Docker run in order to get zombie processes reaped" This reverts commit c972026385ba2d98df4548f24608b0ca62631505. --- hack/libhack/challenge.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hack/libhack/challenge.sh b/hack/libhack/challenge.sh index 0c93027..4d4cd95 100644 --- a/hack/libhack/challenge.sh +++ b/hack/libhack/challenge.sh @@ -44,8 +44,7 @@ challenge::run() { mount_volumes="${mount_baseimage:-} ${mount_challenge}" fi popd - docker run --init \ - --rm \ + docker run --rm \ -p ${CHALLENGE_PORT}:${CHALLENGE_PORT} \ ${mount_volumes:-} \ ${@:-} \ From 21cfaebe8a6c1061036b34cf2d988039a617d8db Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Tue, 27 Aug 2019 13:08:41 +0200 Subject: [PATCH 63/68] Instantiate FrontendReadyHandler --- solvable/src/event_handler_main.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 0a4a0d0..fddd5f3 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -7,7 +7,7 @@ from tfw.fsm import YamlFSM from tfw.event_handlers import FSMAwareEventHandler, ControlEventHandler from tfw.components.ide import IdeHandler, DeployHandler from tfw.components.terminal import TerminalHandler -from tfw.components.frontend import FrontendProxyHandler, ConsoleLogsHandler, MessageQueueHandler +from tfw.components.frontend import FrontendProxyHandler, FrontendReadyHandler, ConsoleLogsHandler, MessageQueueHandler from tfw.components.process_management import ProcessHandler, ProcessLogHandler from tfw.components.fsm import FSMHandler from tfw.main import EventHandlerFactory, setup_logger, setup_signal_handlers @@ -18,6 +18,7 @@ from custom_handlers import CenatorHandler, TestCommandsHandler, messageFSMSteps LOG = logging.getLogger(__name__) + def main(): # pylint: disable=unused-variable setup_logger(__file__) @@ -30,8 +31,7 @@ def main(): YamlFSM, 'test_fsm.yml', {} # jinja2 variables, empty dict enables jinja2 without any variables - ), - initial_trigger='step_1' + ) ), event_handler_type=ControlEventHandler) # Web IDE backend ide_eh = eh_factory.build(IdeHandler( @@ -57,6 +57,9 @@ def main(): ), event_handler_type=ControlEventHandler) # Proxies frontend API calls to frontend frontend_eh = eh_factory.build(FrontendProxyHandler(), event_handler_type=ControlEventHandler) + ready = FrontendReadyHandler('step_1') + ready_eh = eh_factory.build(ready) + ready.stop = ready_eh.stop message_queue_eh = eh_factory.build(MessageQueueHandler(25), event_handler_type=ControlEventHandler) # Writes live logs to console on frontend console_logs_eh = eh_factory.build(ConsoleLogsHandler(stream='stdout')) From b4ece4fc512782213bdb7782beff63bdcbeeae2f Mon Sep 17 00:00:00 2001 From: "R. Richard" Date: Wed, 28 Aug 2019 16:31:06 +0200 Subject: [PATCH 64/68] Refactor API message format --- solvable/src/custom_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solvable/src/custom_handlers.py b/solvable/src/custom_handlers.py index 1d022fb..4c5c0d3 100644 --- a/solvable/src/custom_handlers.py +++ b/solvable/src/custom_handlers.py @@ -12,7 +12,7 @@ class CenatorHandler: keys = ['history.bash'] def handle_event(self, message, connector): # pylint: disable=no-self-use - command = message['value'] + command = message['command'] LOG.debug('User executed command: "%s"', command) MessageSender(connector).send(f'You\'ve executed "{command}"', originator='JOHN CENA') From d9fa35767289e76352b7fac2c2645cb1deef93dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 30 Aug 2019 14:46:15 +0200 Subject: [PATCH 65/68] Remove now duplicate EventHandler type specifications --- solvable/src/event_handler_main.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index fddd5f3..c06def8 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -4,7 +4,7 @@ from functools import partial from tornado.ioloop import IOLoop from tfw.fsm import YamlFSM -from tfw.event_handlers import FSMAwareEventHandler, ControlEventHandler +from tfw.event_handlers import FSMAwareEventHandler from tfw.components.ide import IdeHandler, DeployHandler from tfw.components.terminal import TerminalHandler from tfw.components.frontend import FrontendProxyHandler, FrontendReadyHandler, ConsoleLogsHandler, MessageQueueHandler @@ -32,11 +32,11 @@ def main(): 'test_fsm.yml', {} # jinja2 variables, empty dict enables jinja2 without any variables ) - ), event_handler_type=ControlEventHandler) + )) # Web IDE backend ide_eh = eh_factory.build(IdeHandler( patterns=['/home/user/workdir/*', '/srv/webservice/user_ops.py'] - ), event_handler_type=ControlEventHandler) + )) deploy_eh = eh_factory.build(DeployHandler()) # Web shell backend terminal_eh = eh_factory.build(TerminalHandler( @@ -44,23 +44,25 @@ def main(): user=TAOENV.USER, working_directory=TFWENV.TERMINADO_WD, histfile=TFWENV.HISTFILE - ), event_handler_type=ControlEventHandler) + )) # Handles 'deploy' button clicks - processmanager_eh = eh_factory.build(ProcessHandler( + process_eh = eh_factory.build(ProcessHandler( supervisor_uri=TFWENV.SUPERVISOR_HTTP_URI - ), event_handler_type=ControlEventHandler) + )) # Sends live logs of webservice process to frontend - logmonitor_eh = eh_factory.build(ProcessLogHandler( + processlog_eh = eh_factory.build(ProcessLogHandler( process_name='webservice', supervisor_uri=TFWENV.SUPERVISOR_HTTP_URI, log_tail=2000 - ), event_handler_type=ControlEventHandler) + )) # Proxies frontend API calls to frontend - frontend_eh = eh_factory.build(FrontendProxyHandler(), event_handler_type=ControlEventHandler) + frontendproxy_eh = eh_factory.build(FrontendProxyHandler()) + # Initiates first FSM step ready = FrontendReadyHandler('step_1') ready_eh = eh_factory.build(ready) ready.stop = ready_eh.stop - message_queue_eh = eh_factory.build(MessageQueueHandler(25), event_handler_type=ControlEventHandler) + # Manages message queues + messagequeue_eh = eh_factory.build(MessageQueueHandler(25)) # Writes live logs to console on frontend console_logs_eh = eh_factory.build(ConsoleLogsHandler(stream='stdout')) From 8c2cd7e44ee4cf37a863e20db684e9e8925184c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 30 Aug 2019 16:41:59 +0200 Subject: [PATCH 66/68] Ignore trailing slash in controller route --- controller/opt/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controller/opt/server.py b/controller/opt/server.py index 712c1df..6a1265e 100644 --- a/controller/opt/server.py +++ b/controller/opt/server.py @@ -26,8 +26,9 @@ if __name__ == '__main__': event_handler_type=FSMAwareEventHandler ) + route = os.environ['SECRET'].rstrip('/') application = Application([( - f'/{os.environ["SECRET"]}', + f'/{route}/?', ControllerPostHandler, {'controller': controller_eh} )]) From e016fc0a248ff1281a8f9211e687d60bf87ed44a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 30 Aug 2019 17:38:27 +0200 Subject: [PATCH 67/68] Configure frontend on 'frontend.ready' event --- solvable/src/event_handler_main.py | 17 ++++++++++++----- solvable/src/frontend_config.yaml | 27 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 solvable/src/frontend_config.yaml diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index c06def8..bca9455 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -1,4 +1,5 @@ import logging +from os.path import dirname, realpath, join from functools import partial from tornado.ioloop import IOLoop @@ -7,7 +8,8 @@ from tfw.fsm import YamlFSM from tfw.event_handlers import FSMAwareEventHandler from tfw.components.ide import IdeHandler, DeployHandler from tfw.components.terminal import TerminalHandler -from tfw.components.frontend import FrontendProxyHandler, FrontendReadyHandler, ConsoleLogsHandler, MessageQueueHandler +from tfw.components.frontend import FrontendProxyHandler, FrontendReadyHandler +from tfw.components.frontend import ConsoleLogsHandler, MessageQueueHandler, FrontendConfigHandler from tfw.components.process_management import ProcessHandler, ProcessLogHandler from tfw.components.fsm import FSMHandler from tfw.main import EventHandlerFactory, setup_logger, setup_signal_handlers @@ -16,11 +18,12 @@ from tfw.config import TFWENV, TAOENV from custom_handlers import CenatorHandler, TestCommandsHandler, messageFSMStepsHandler LOG = logging.getLogger(__name__) +here = dirname(realpath(__file__)) def main(): - # pylint: disable=unused-variable + # pylint: disable=unused-variable,too-many-locals setup_logger(__file__) eh_factory = EventHandlerFactory() @@ -58,9 +61,13 @@ def main(): # Proxies frontend API calls to frontend frontendproxy_eh = eh_factory.build(FrontendProxyHandler()) # Initiates first FSM step - ready = FrontendReadyHandler('step_1') - ready_eh = eh_factory.build(ready) - ready.stop = ready_eh.stop + frontendready = FrontendReadyHandler('step_1') + frontendready_eh = eh_factory.build(frontendready) + frontendready.stop = frontendready_eh.stop + # Configures frontend + frontendconfig_eh = eh_factory.build( + FrontendConfigHandler(join(here, 'frontend_config.yaml')) + ) # Manages message queues messagequeue_eh = eh_factory.build(MessageQueueHandler(25)) # Writes live logs to console on frontend diff --git a/solvable/src/frontend_config.yaml b/solvable/src/frontend_config.yaml new file mode 100644 index 0000000..f927833 --- /dev/null +++ b/solvable/src/frontend_config.yaml @@ -0,0 +1,27 @@ +dashboard: + layout: terminal-ide-web + hideMessages: false + iframeUrl: /webservice + showUrlBar: false + terminalMenuItem: terminal + reloadIframeOnDeploy: false + enabledLayouts: + - terminal-ide-web + - terminal-ide-vertical + - terminal-web + - ide-web-vertical + - terminal-ide-horizontal + - terminal-only + - ide-only + - web-only +ide: + autoSaveInterval: 444 + showDeployButton: true + deployButtonText: + TODEPLOY: Deploy + DEPLOYED: Deployed + DEPLOYING: Reloading app... + FAILED: Deployment failed +site: + askReloadSite: false + documentTitle: Avatao Tutorials From fa1cdbfd6ef38539cb6615af5c5f387dbc52d6f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 6 Sep 2019 15:56:35 +0200 Subject: [PATCH 68/68] Open /tmp/tfw_* pipes for all users --- solvable/src/pipe_io_main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solvable/src/pipe_io_main.py b/solvable/src/pipe_io_main.py index c240130..5b3bfeb 100644 --- a/solvable/src/pipe_io_main.py +++ b/solvable/src/pipe_io_main.py @@ -17,7 +17,8 @@ def main(): json_pipe_eh = eh_factory.build(PipeIOHandler( '/tmp/tfw_send', - '/tmp/tfw_recv' + '/tmp/tfw_recv', + permissions=0o666 )) proxy_pipe_eh = eh_factory.build(ProxyPipeConnectorHandler(TFWENV.PIPES_DIR))