From 8fab3d22262257a7675d9eac1f93a35bf2b27564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 24 Jul 2018 17:17:14 +0200 Subject: [PATCH 01/12] Conform new fsm_update API --- controller/opt/server.py | 2 +- solvable/src/event_handler_main.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/controller/opt/server.py b/controller/opt/server.py index 82a3811..8155726 100644 --- a/controller/opt/server.py +++ b/controller/opt/server.py @@ -15,7 +15,7 @@ class ControllerPostHandler(RequestHandler): def post(self, *args, **kwargs): self.set_header('Content-Type', 'application/json') self.write(json.dumps({ - 'solved': self.controller.in_accepted_state + 'solved': self.controller.fsm_in_accepted_state })) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 9ff99e5..36a8902 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -86,14 +86,15 @@ class MessageFSMStepsEventHandler(FSMAwareEventHandler): def handle_event(self, message): pass - def handle_fsm_step(self, from_state, to_state, trigger): + def handle_fsm_step(self, **kwargs): """ When the FSM steps this method is invoked. + Receives a 'data' field from an fsm_update message as kwargs. """ MessageSender().send( 'FSM info', - f'FSM has stepped from state "{from_state}" ' - f'to state "{to_state}" in response to trigger "{trigger}"' + f'FSM has stepped from state "{kwargs["last_event"]["from_state"]}" ' + f'to state "{kwargs["current_state"]}" in response to trigger "{kwargs["last_event"]["trigger"]}"' ) From 265560b6144c4dc8280a3d5e4815dcaea279e14f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Thu, 26 Jul 2018 14:00:24 +0200 Subject: [PATCH 02/12] Conform new import routes --- controller/Dockerfile | 1 - controller/opt/server.py | 2 +- solvable/src/event_handler_main.py | 3 ++- solvable/src/test_fsm.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/controller/Dockerfile b/controller/Dockerfile index ef96d41..d09b2fd 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -7,7 +7,6 @@ ENV PYTHONPATH="/usr/local/lib" \ TFW_AUTH_KEY="/tmp/tfw-auth.key" \ CONTROLLER_PORT=5555 -RUN pip3 install watchdog transitions COPY ./controller/ / CMD ["python3", "/opt/server.py"] diff --git a/controller/opt/server.py b/controller/opt/server.py index 8155726..175d738 100644 --- a/controller/opt/server.py +++ b/controller/opt/server.py @@ -4,7 +4,7 @@ import json from tornado.ioloop import IOLoop from tornado.web import RequestHandler, Application -from tfw import FSMAwareEventHandler +from tfw.event_handler_base import FSMAwareEventHandler class ControllerPostHandler(RequestHandler): diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 36a8902..1d09f7a 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -3,7 +3,8 @@ from functools import partial from tornado.ioloop import IOLoop -from tfw import YamlFSM, FSMAwareEventHandler +from tfw.fsm import YamlFSM +from tfw.event_handler_base import FSMAwareEventHandler from tfw.components import IdeEventHandler, TerminalEventHandler from tfw.components import ProcessManagingEventHandler, BashMonitor from tfw.components import TerminalCommands, LogMonitoringEventHandler diff --git a/solvable/src/test_fsm.py b/solvable/src/test_fsm.py index 3dcf7c3..2db9b97 100644 --- a/solvable/src/test_fsm.py +++ b/solvable/src/test_fsm.py @@ -2,7 +2,7 @@ from os.path import exists -from tfw import LinearFSM +from tfw.fsm import LinearFSM from tfw.networking import MessageSender From 8b47b301e4d646f881c00cb64fe5f65e705f1fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 27 Jul 2018 13:59:14 +0200 Subject: [PATCH 03/12] Use jinja2 in default YamlFSM --- solvable/src/event_handler_main.py | 6 +++++- solvable/src/test_fsm.yml | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 1d09f7a..a3639e2 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -103,7 +103,11 @@ if __name__ == '__main__': # TFW component EventHandlers (builtins, required for their respective functionalities) fsm = FSMManagingEventHandler( # TFW FSM key='fsm', - fsm_type=partial(YamlFSM, 'test_fsm.yml') + fsm_type=partial( + YamlFSM, + 'test_fsm.yml', + {} # jinja2 variables, use empty dict to enable jinja2 parsing without any variables + ) ) ide = IdeEventHandler( # Web IDE backend key='ide', diff --git a/solvable/src/test_fsm.yml b/solvable/src/test_fsm.yml index b195479..d34ebcb 100644 --- a/solvable/src/test_fsm.yml +++ b/solvable/src/test_fsm.yml @@ -41,3 +41,8 @@ transitions: - trigger: step_5 source: '4' dest: '5' + {% for i in range(5) %} # you can also use jinja2 in this config file + - trigger: 'step_next' + source: '{{i}}' + dest: '{{i+1}}' + {% endfor %} From db551e87883d848e1b18f8d2d3a87d6ee41bc7ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 27 Jul 2018 13:59:34 +0200 Subject: [PATCH 04/12] Add notice on YamlFSM jinja2 support --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9a8e647..16acabf 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ A good state machine is the backbone of a good TFW challenge. There are two ways to define a state machine: - Using a YAML configuration file - - Implementing it in Python by hand + - Implementing it in Python The first option allows you to handle FSM callbacks and custom logic in any programming language (not just Python) and is generally really easy to work with (you can execute arbitrary shell commands on events). You should choose this method unless you have good reason not to. @@ -230,6 +230,8 @@ It is also possible to add preconditions to transitions. This is done by adding a `predicates` key with a list of shell commands to run. If you do this, the transition will only succeed if the return code of all predicates was `0` (as per unix convention for success). +Our `YamlFSM` implementation also supports jinja2 templates inside the `YAML` config file (examples in `test_fsm.yml`). + ## Baby steps When creating your own challenge the process should be the following: From 5e390e1449a97d590e948bb04b9efa8ce2185902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 30 Jul 2018 08:56:40 +0200 Subject: [PATCH 05/12] Update lies in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 16acabf..b9e1e52 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ Refer to the example in this repo. ### src -This folder contains the source code of a server running TFW and an other server running our event handlers. +This folder contains the source code of our pre-written event handlers and example FSMs. Note that this is not a part of the framework by any means, these are just simple examples. ``` From 38963e7b81d80b1e60fff62173c6c2de4297ea11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 1 Aug 2018 15:05:08 +0200 Subject: [PATCH 06/12] Improve Dockerfile --- solvable/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/solvable/Dockerfile b/solvable/Dockerfile index 7dd51b0..aa6382f 100644 --- a/solvable/Dockerfile +++ b/solvable/Dockerfile @@ -7,9 +7,9 @@ RUN pip3 install Flask==1.0 \ git+https://github.com/avatao-content/tfwconnector.git#subdirectory=python3 # Define variables to use later -ENV TFW_EHMAIN_DIR="/srv/.tfw_builtin_ehs" \ - TFW_WEBSERVICE_DIR="/srv/webservice" \ - TFW_IDE_WD="/home/${AVATAO_USER}/workdir" \ +ENV TFW_EHMAIN_DIR="${TFW_DIR}/builtin_event_handlers" \ + TFW_WEBSERVICE_DIR="/srv/webservice" \ + TFW_IDE_WD="/home/${AVATAO_USER}/workdir" \ TFW_TERMINADO_WD="/home/${AVATAO_USER}/workdir" # Copy TFW related stuff to a dedicated directory @@ -26,7 +26,8 @@ RUN mkdir -p ${TFW_IDE_WD} &&\ chmod -R 755 "${TFW_IDE_WD}" "${TFW_WEBSERVICE_DIR}" # Hide TFW related code from user -RUN chown -R root:root ${TFW_SERVER_DIR} && chmod -R 700 ${TFW_SERVER_DIR} +RUN chown -R root:root ${TFW_SERVER_DIR} ${TFW_DIR} &&\ + chmod -R 700 ${TFW_SERVER_DIR} ${TFW_DIR} # Make AVATAO_USER's home writeable and set it as WORKDIR # Make webservice directory writable From 829c2448675c02bb5eb86183176aa388460c9b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 3 Aug 2018 13:33:11 +0200 Subject: [PATCH 07/12] Fix HOTRELOAD --- hack/libhack/challenge.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/libhack/challenge.sh b/hack/libhack/challenge.sh index 8a7e558..4ed1377 100644 --- a/hack/libhack/challenge.sh +++ b/hack/libhack/challenge.sh @@ -40,7 +40,7 @@ challenge::run() { if [[ -d "${BASEIMAGE_PATH}" ]]; then mount_baseimage="-v ${BASEIMAGE_PATH}/lib/tfw:/usr/local/lib/tfw" fi - mount_challenge="-v ${CHALLENGE_PATH}/solvable/src:/srv/.tfw_builtin_ehs" + mount_challenge="-v ${CHALLENGE_PATH}/solvable/src:/.tfw/builtin_event_handlers" mount_volumes="${mount_baseimage:-} ${mount_challenge}" fi popd From 3bda7bc5403e0b7211ce8abc5cf08df9ea4ba957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 7 Aug 2018 17:01:58 +0200 Subject: [PATCH 08/12] Make cleanup() calls automatic in event_handler_main.py --- solvable/src/event_handler_main.py | 34 ++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index a3639e2..4b0b564 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -1,10 +1,12 @@ from ast import literal_eval from functools import partial +from signal import signal, SIGTERM, SIGINT, SIGHUP +from sys import exit as sysexit from tornado.ioloop import IOLoop from tfw.fsm import YamlFSM -from tfw.event_handler_base import FSMAwareEventHandler +from tfw.event_handler_base import EventHandlerBase, FSMAwareEventHandler from tfw.components import IdeEventHandler, TerminalEventHandler from tfw.components import ProcessManagingEventHandler, BashMonitor from tfw.components import TerminalCommands, LogMonitoringEventHandler @@ -99,9 +101,11 @@ class MessageFSMStepsEventHandler(FSMAwareEventHandler): ) -if __name__ == '__main__': +def main(): + # pylint: disable=possibly-unused-variable + # # TFW component EventHandlers (builtins, required for their respective functionalities) - fsm = FSMManagingEventHandler( # TFW FSM + fsm = FSMManagingEventHandler( # TFW FSM key='fsm', fsm_type=partial( YamlFSM, @@ -131,20 +135,32 @@ if __name__ == '__main__': ) # Your custom event handlers - message_fsm_steps = MessageFSMStepsEventHandler( + message_fsm_steps_eh = MessageFSMStepsEventHandler( key='test' ) # Terminal command handlers commands = TestCommands(bashrc=f'/home/{TAOENV.USER}/.bashrc') terminal.historymonitor.subscribe_callback(commands.callback) - - # Example terminal command callback terminal.historymonitor.subscribe_callback(cenator) + # Raise SystemExit on signals we should handle gracefully (make finally run) + trigger_finally = lambda a, b: sysexit() + signal(SIGTERM, trigger_finally) + signal(SIGINT, trigger_finally) + signal(SIGHUP, trigger_finally) + + # Start event handlers and clean up on stopping try: IOLoop.instance().start() finally: - eventhandlers = {fsm, ide, terminal, processmanager, logmonitor, message_fsm_steps} - for eh in eventhandlers: - eh.cleanup() + event_handlers = ( + eh for eh in locals().values() + if isinstance(eh, EventHandlerBase) + ) + for event_handler in event_handlers: + event_handler.cleanup() + + +if __name__ == '__main__': + main() From e7d78ed289492bd9745d52826a40157cb6c848c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 7 Aug 2018 17:03:41 +0200 Subject: [PATCH 09/12] Add snapshot event handler --- solvable/src/event_handler_main.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 4b0b564..b1702d3 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -10,7 +10,7 @@ from tfw.event_handler_base import EventHandlerBase, FSMAwareEventHandler from tfw.components import IdeEventHandler, TerminalEventHandler from tfw.components import ProcessManagingEventHandler, BashMonitor from tfw.components import TerminalCommands, LogMonitoringEventHandler -from tfw.components import FSMManagingEventHandler +from tfw.components import FSMManagingEventHandler, DirectorySnapshottingEventHandler from tfw.networking import MessageSender, TFWServerConnector from tfw.config import TFWENV from tfw.config.logs import logging @@ -133,6 +133,13 @@ def main(): process_name='webservice', log_tail=2000 ) + snapshot = DirectorySnapshottingEventHandler( # Manages filesystem snapshots of directories + key='snapshot', + directories=[ + TFWENV.IDE_WD, + TFWENV.WEBSERVICE_DIR + ] + ) # Your custom event handlers message_fsm_steps_eh = MessageFSMStepsEventHandler( From 4a1073e5246013192f19365e7bf5466bf610581c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 31 Aug 2018 16:57:00 +0200 Subject: [PATCH 10/12] Bootify webservice db session handling --- solvable/src/webservice/model.py | 35 ++++++++++++++++++++----------- solvable/src/webservice/server.py | 24 ++++++++++----------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/solvable/src/webservice/model.py b/solvable/src/webservice/model.py index d722cdc..b084b1e 100644 --- a/solvable/src/webservice/model.py +++ b/solvable/src/webservice/model.py @@ -13,18 +13,29 @@ session_factory = sessionmaker( ) -@contextmanager -def Session(factory=session_factory): - session = factory() - try: - yield session - session.commit() - except: - session.rollback() - raise - # session is closed by flask - # finally: - # session.close() +class SessionWrapper: + def __init__(self): + self._session_factory = session_factory + self._session_handle = None + + @contextmanager + def session(self): + try: + yield self._session + self._session.commit() + except: + self._session.rollback() + raise + + @property + def _session(self): + if self._session_handle is None: + self._session_handle = self._session_factory() + return self._session_handle + + def teardown(self): + if self._session_handle is not None: + self._session_handle.close() Base = declarative_base() diff --git a/solvable/src/webservice/server.py b/solvable/src/webservice/server.py index c359c61..1b36762 100644 --- a/solvable/src/webservice/server.py +++ b/solvable/src/webservice/server.py @@ -1,9 +1,8 @@ from os import urandom, getenv -from functools import partial from flask import Flask, render_template, request, session, url_for, g -from model import init_db, session_factory, Session +from model import init_db, SessionWrapper from user_ops import UserOps from errors import InvalidCredentialsError, UserExistsError @@ -16,25 +15,24 @@ app.jinja_env.globals.update( # pylint: disable=no-member ) -def get_db_session(): - if not hasattr(g, 'db_session'): - g.db_session = session_factory() - return g.db_session - -Session = partial(Session, get_db_session) +@app.before_request +def setup_db(): + # pylint: disable=protected-access + g._db_session_wrapper = SessionWrapper() + g.db_session = g._db_session_wrapper.session @app.teardown_appcontext -def close_db_session(err): # pylint: disable=unused-argument - if hasattr(g, 'db_session'): - g.db_session.close() +def close_db_session(_): + # pylint: disable=protected-access + g._db_session_wrapper.teardown() @app.route('/', methods=['GET', 'POST']) def index(): if request.method == 'POST': try: - with Session() as db_session: + with g.db_session() as db_session: UserOps( request.form.get('username'), request.form.get('password'), @@ -67,7 +65,7 @@ def register(): return render_template('register.html', alert='Passwords do not match! Please try again.') try: - with Session() as db_session: + with g.db_session() as db_session: UserOps( request.form.get('username'), request.form.get('password'), From 97c593640d41208399cb1fd02e720d9022d27544 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 17 Sep 2018 11:49:10 +0200 Subject: [PATCH 11/12] Add macOS grep aliases to hack scripts --- hack/bootstrap.sh | 2 +- hack/tfw.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/bootstrap.sh b/hack/bootstrap.sh index 643a9cc..a6aedfa 100755 --- a/hack/bootstrap.sh +++ b/hack/bootstrap.sh @@ -3,7 +3,7 @@ set -eu set -o pipefail set -o errtrace shopt -s expand_aliases -[ "$(uname)" == "Darwin" ] && alias sed="gsed" || : +[ "$(uname)" == "Darwin" ] && alias sed="gsed" && alias grep="ggrep" || : HERE="$(pwd)" CHALLENGE=${CHALLENGE:-test-tutorial-framework} diff --git a/hack/tfw.sh b/hack/tfw.sh index 46a843b..f05f2b7 100755 --- a/hack/tfw.sh +++ b/hack/tfw.sh @@ -3,7 +3,7 @@ set -eu set -o pipefail set -o errtrace shopt -s expand_aliases -[ "$(uname)" == "Darwin" ] && alias readlink="greadlink" || : +[ "$(uname)" == "Darwin" ] && alias readlink="greadlink" && alias grep="ggrep" || : SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" TFW_PATH="${TFW_PATH:-$SCRIPT_DIR/../..}" From 89c55651269edfc12cdc14b2535d70b7e4b7604b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 13 Feb 2019 13:51:55 +0100 Subject: [PATCH 12/12] Add missing GNU utils to bootstrap script dep list --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index b9e1e52..4b69354 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,9 @@ Dependencies: - yarn - Angular CLI - GNU coreutils +- GNU findutils +- GNU sed +- GNU grep Just copy and paste the following command in a terminal: