From 9b3e62852d3e807e0ff0c955fc3c4101cee6adef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 29 Jun 2018 22:08:05 +0200 Subject: [PATCH 1/6] Move FSM initialization logic to event_handler_main.py --- solvable/src/event_handler_main.py | 11 ++++++++++- solvable/src/tfw_server.py | 10 +--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index 297f132..e580a9e 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -1,15 +1,19 @@ from ast import literal_eval +from functools import partial from tornado.ioloop import IOLoop from tfw.components import IdeEventHandler, TerminalEventHandler from tfw.components import ProcessManagingEventHandler, BashMonitor from tfw.components import TerminalCommands, LogMonitoringEventHandler +from tfw.components import FSMManagingEventHandler from tfw.networking import MessageSender, TFWServerConnector from tfw.config import TFWENV from tfw.config.logs import logging from tao.config import TAOENV +from test_fsm import TestFSM + LOG = logging.getLogger(__name__) @@ -76,6 +80,11 @@ class TestCommands(TerminalCommands): if __name__ == '__main__': + FiveStepTestFSM = partial(TestFSM, 5) + fsm = FSMManagingEventHandler( # TFW FSM + key='fsm', + fsm_type=FiveStepTestFSM + ) ide = IdeEventHandler( # Web IDE backend key='ide', allowed_directories=[TFWENV.IDE_WD, TFWENV.WEBSERVICE_DIR], @@ -96,7 +105,7 @@ if __name__ == '__main__': process_name='webservice', log_tail=2000 ) - eventhandlers = {ide, terminal, processmanager, logmonitor} + eventhandlers = {ide, terminal, processmanager, logmonitor, fsm} commands = TestCommands(bashrc=f'/home/{TAOENV.USER}/.bashrc') terminal.historymonitor.subscribe_callback(commands.callback) diff --git a/solvable/src/tfw_server.py b/solvable/src/tfw_server.py index b202e92..78766ac 100644 --- a/solvable/src/tfw_server.py +++ b/solvable/src/tfw_server.py @@ -1,17 +1,9 @@ -from functools import partial - from tornado.ioloop import IOLoop from tfw.networking import TFWServer from tfw.config import TFWENV -from tfw.config.logs import logging -from test_fsm import TestFSM - -LOG = logging.getLogger(__name__) if __name__ == '__main__': - FiveStepTestFSM = partial(TestFSM, 5) - TFWServer(FiveStepTestFSM).listen(TFWENV.WEB_PORT) - + TFWServer().listen(TFWENV.WEB_PORT) IOLoop.instance().start() From 6b1fa664c48fe5a49f8e56320094f8d9715e2fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 29 Jun 2018 22:52:39 +0200 Subject: [PATCH 2/6] Remove TFWServer logic (handled in baseimage from now) --- solvable/Dockerfile | 4 ++-- solvable/src/tfw_server.py | 9 --------- solvable/supervisor/event_handler_main.conf | 2 +- solvable/supervisor/tfw_server.conf | 4 ---- 4 files changed, 3 insertions(+), 16 deletions(-) delete mode 100644 solvable/src/tfw_server.py delete mode 100644 solvable/supervisor/tfw_server.conf diff --git a/solvable/Dockerfile b/solvable/Dockerfile index 10f08d2..7dd51b0 100644 --- a/solvable/Dockerfile +++ b/solvable/Dockerfile @@ -7,13 +7,13 @@ RUN pip3 install Flask==1.0 \ git+https://github.com/avatao-content/tfwconnector.git#subdirectory=python3 # Define variables to use later -ENV TFW_SERVER_DIR="/srv/.tfw" \ +ENV TFW_EHMAIN_DIR="/srv/.tfw_builtin_ehs" \ TFW_WEBSERVICE_DIR="/srv/webservice" \ TFW_IDE_WD="/home/${AVATAO_USER}/workdir" \ TFW_TERMINADO_WD="/home/${AVATAO_USER}/workdir" # Copy TFW related stuff to a dedicated directory -COPY solvable/src ${TFW_SERVER_DIR}/ +COPY solvable/src ${TFW_EHMAIN_DIR}/ # Copy webservice to a dedicated directory COPY solvable/src/webservice/ ${TFW_WEBSERVICE_DIR}/ diff --git a/solvable/src/tfw_server.py b/solvable/src/tfw_server.py deleted file mode 100644 index 78766ac..0000000 --- a/solvable/src/tfw_server.py +++ /dev/null @@ -1,9 +0,0 @@ -from tornado.ioloop import IOLoop - -from tfw.networking import TFWServer -from tfw.config import TFWENV - - -if __name__ == '__main__': - TFWServer().listen(TFWENV.WEB_PORT) - IOLoop.instance().start() diff --git a/solvable/supervisor/event_handler_main.conf b/solvable/supervisor/event_handler_main.conf index ab56af2..021e13c 100644 --- a/solvable/supervisor/event_handler_main.conf +++ b/solvable/supervisor/event_handler_main.conf @@ -1,4 +1,4 @@ [program:event_handler_main] user=root -directory=%(ENV_TFW_SERVER_DIR)s +directory=%(ENV_TFW_EHMAIN_DIR)s command=python3 event_handler_main.py diff --git a/solvable/supervisor/tfw_server.conf b/solvable/supervisor/tfw_server.conf deleted file mode 100644 index 1422371..0000000 --- a/solvable/supervisor/tfw_server.conf +++ /dev/null @@ -1,4 +0,0 @@ -[program:tfwserver] -user=root -directory=%(ENV_TFW_SERVER_DIR)s -command=python3 tfw_server.py \ No newline at end of file From 98f297cff29df983d8f6ee7054257ba1847348d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 10 Jul 2018 17:32:43 +0200 Subject: [PATCH 3/6] Make python and yaml FSMs eqvivalent so it's a nice example --- solvable/src/event_handler_main.py | 6 ++--- solvable/src/test_fsm.py | 29 +++++++++++++------- solvable/src/test_fsm.yml | 43 ++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 solvable/src/test_fsm.yml diff --git a/solvable/src/event_handler_main.py b/solvable/src/event_handler_main.py index e580a9e..fb28b21 100644 --- a/solvable/src/event_handler_main.py +++ b/solvable/src/event_handler_main.py @@ -3,6 +3,7 @@ from functools import partial from tornado.ioloop import IOLoop +from tfw import YamlFSM from tfw.components import IdeEventHandler, TerminalEventHandler from tfw.components import ProcessManagingEventHandler, BashMonitor from tfw.components import TerminalCommands, LogMonitoringEventHandler @@ -80,10 +81,9 @@ class TestCommands(TerminalCommands): if __name__ == '__main__': - FiveStepTestFSM = partial(TestFSM, 5) fsm = FSMManagingEventHandler( # TFW FSM key='fsm', - fsm_type=FiveStepTestFSM + fsm_type=partial(YamlFSM, 'test_fsm.yml') ) ide = IdeEventHandler( # Web IDE backend key='ide', @@ -105,7 +105,7 @@ if __name__ == '__main__': process_name='webservice', log_tail=2000 ) - eventhandlers = {ide, terminal, processmanager, logmonitor, fsm} + eventhandlers = {fsm, ide, terminal, processmanager, logmonitor} commands = TestCommands(bashrc=f'/home/{TAOENV.USER}/.bashrc') terminal.historymonitor.subscribe_callback(commands.callback) diff --git a/solvable/src/test_fsm.py b/solvable/src/test_fsm.py index d071ff4..3dcf7c3 100644 --- a/solvable/src/test_fsm.py +++ b/solvable/src/test_fsm.py @@ -1,3 +1,7 @@ +# This defines an FSM equvivalent to test_fsm.yml + +from os.path import exists + from tfw import LinearFSM from tfw.networking import MessageSender @@ -5,24 +9,29 @@ from tfw.networking import MessageSender class TestFSM(LinearFSM): # pylint: disable=unused-argument - def __init__(self, number_of_steps): - super().__init__(number_of_steps) + def __init__(self): + super().__init__(6) self.message_sender = MessageSender() + self.subscribe_predicate('step_3', self.step_3_allowed) + + @staticmethod + def step_3_allowed(): + return exists('/home/user/workdir/allow_step_3') def on_enter_1(self, event_data): - self.state_notify(1) + self.message_sender.send('FSM', 'Entered state 1!') def on_enter_2(self, event_data): - self.state_notify(2) + filename = '/home/user/workdir/cat.txt' + with open(filename, 'w') as ofile: + ofile.write('As you can see it is possible to write arbitrary python code here.') + self.message_sender.send('FSM', f'Entered state 2! Written stuff to {filename}') def on_enter_3(self, event_data): - self.state_notify(3) + self.message_sender.send('FSM', 'Entered state 3!') def on_enter_4(self, event_data): - self.state_notify(4) + self.message_sender.send('FSM', 'Entered state 4!') def on_enter_5(self, event_data): - self.state_notify(5) - - def state_notify(self, state): - self.message_sender.send('TestFSM', f'Entered state {state}!') + self.message_sender.send('FSM', 'Entered state 5!') diff --git a/solvable/src/test_fsm.yml b/solvable/src/test_fsm.yml new file mode 100644 index 0000000..b195479 --- /dev/null +++ b/solvable/src/test_fsm.yml @@ -0,0 +1,43 @@ +# This defines an FSM equvivalent to test_fsm.py + +states: + - name: '0' + - name: '1' + on_enter: | + python3 -c "from tfwconnector import MessageSender; MessageSender().send('FSM', 'Entered state 1!')" + - name: '2' + on_enter: | + file=/home/user/workdir/cat.txt + echo "As you can see it is possible to execute arbitrary shell commands here." >> $file + python3 -c \ + " + from tfwconnector import MessageSender + MessageSender().send('FSM', 'Entered state 2! Written stuff to $file') + " + - name: '3' + on_enter: | + python3 -c "from tfwconnector import MessageSender; MessageSender().send('FSM', 'Entered state 3!')" + - name: '4' + on_enter: | + python3 -c "from tfwconnector import MessageSender; MessageSender().send('FSM', 'Entered state 4!')" + - name: '5' + on_enter: | + python3 -c "from tfwconnector import MessageSender; MessageSender().send('FSM', 'Entered state 5!')" +transitions: + - trigger: step_1 + source: '0' + dest: '1' + - trigger: step_2 + source: '1' + dest: '2' + - trigger: step_3 + source: '2' + dest: '3' + predicates: + - '[ -f /home/user/workdir/allow_step_3 ]' # in bash -f means that the file exists + - trigger: step_4 + source: '3' + dest: '4' + - trigger: step_5 + source: '4' + dest: '5' From c9439f547bd7f86947d74abf1926aa7a0ea9ca55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 10 Jul 2018 17:42:08 +0200 Subject: [PATCH 4/6] Remove lies from docs and update info --- README.md | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index d9a76fb..198ee33 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ To do this simply issue `BASEIMAGE_ONLY=version bash -c "$(curl -fsSL https://gi The repository of a tutorial-framework based challenge is quite similar to a regular challenge. The project root should look something like this: -```text +``` your_repo ├── solvable │ └── [TFW based Docker image] @@ -112,7 +112,7 @@ From now on we are going to focus on the `solvable` image. Let us take a closer look on `solvable`: -```text +``` solvable ├── Dockerfile ├── nginx webserver configurations @@ -134,7 +134,7 @@ This means that in order to listen on more than a single port we must use a reve Any `.conf` files in `solvable/nginx/` will be automatically included in the nginx configuration. In case you want to serve a website or service you must proxy it through `TFW_PUBLIC_PORT`. This is really easy: just create a config file in `solvable/nginx/` similar to this one: -```text +``` location /yoururl { proxy_pass http://127.0.0.1:3333; } @@ -157,7 +157,7 @@ You can even configure your processes to start with the container by including ` To run your own webservice for instance you need to create a config file in `solvable/supervisor/` similar to this one: -```text +``` [program:yourprogram] user=user directory=/home/user/example/ @@ -183,17 +183,12 @@ Refer to the example in this repo. This folder contains the source code of a server running TFW and an other server running our event handlers. Note that this is not a part of the framework by any means, these are just simple examples. -```text -solvable/src -├── tfw_server.py tutorial-framework server -├── event_handler_main.py event handlers implemented in python -└── test_fsm.py example FSM ``` - -The core of the framework is the `TFWServer` class, which is instanciated in `tfw_server.py`. -This class handles the forwarding of the messages from the frontend to the event handlers connecting to it via ZMQ. -It also manages the FSM. -As you can see this file is set up to start with the container in `solvable/supervisor/tfw_server.conf`. +solvable/src +├── event_handler_main.py event handlers implemented in python +├── test_fsm.py example FSM in python +└── test_fsm.yml example FSM in yaml +``` `event_handler_main.py` contains example usage of our pre-defined event handlers written in Python3. As you can see they run in a separate process (set up in `solvable/supervisor/event_handler_main.conf`). @@ -202,6 +197,8 @@ These event handlers could be implemented in any language that has ZMQ bindings. Note that you don't have to use all our event handlers. Should you want to avoid using a feature, you can just delete the appropriate event handler from `event_handler_main.py`. +`test_fsm.yml` and `test_fsm.py` are the implementations of the same FSM in YAML and Python to provide you examples of creating your own machine. + It is genarally a good idea to separate these files from the rest of the stuff in `solvable`, so it is a good practice to create an `src` directory. ## Baby steps @@ -209,7 +206,8 @@ It is genarally a good idea to separate these files from the rest of the stuff i When creating your own challenge the process should be the following: 1. Use our install script to bootstrap your dev environment 2. Create an FSM that describes your challenge - - An example is in `solvable/src/test_fsm.py` + - An example is in `solvable/src/test_fsm.yml` + - The same FSM in python is in `solvable/src/test_fsm.py` 3. Create a `TFWServer` instance and set it up to run: - Create a server app: `solvable/src/tfw_server.py` - Set it up to run: `solvable/supervisor/tfw_server.conf` From 7949d944931ca397b1873fab7b7b5a2110d7d32c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 10 Jul 2018 17:53:45 +0200 Subject: [PATCH 5/6] Update docs with FSM stuff --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 198ee33..503967a 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,29 @@ Should you want to avoid using a feature, you can just delete the appropriate ev It is genarally a good idea to separate these files from the rest of the stuff in `solvable`, so it is a good practice to create an `src` directory. +### FSM + +A good state machine is the backbone of a good TFW challenge. + +There are two ways to define a state machine: + - Using a YAML configuration file + - Implementing it in Python by hand + +The first option enables you to handle FSM callbacks and custom logic in any programming language (not just Python) and is generally really easy to work with. +You should choose this method unless you have good reason not to. +This involves creating your YAML file (see `test_fsm.yml` for an example) and parsing it using our `YamlFSM` class (see `event_handler_main.py` for an example). + +The second option allows you to implement your FSM in Python, using the transitions library. +To do this just subclass our `FSMBase` class or use our `LinearFSM` class for simple machines (see `test_fsm.py` for an example). + +In your FSM you can define callbacks for states and transitions. +State callbacks: + - `on_enter` + - `on_exit` +Transition callbacks: + - `before` + - `after` + ## Baby steps When creating your own challenge the process should be the following: From 185f1165cfab14a85fdeb82a9c078165e73482e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 11 Jul 2018 10:17:19 +0200 Subject: [PATCH 6/6] Update FSM documentation --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 503967a..9a8e647 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,7 @@ There are two ways to define a state machine: - Using a YAML configuration file - Implementing it in Python by hand -The first option enables you to handle FSM callbacks and custom logic in any programming language (not just Python) and is generally really easy to work with. +The first option allows you to handle FSM callbacks and custom logic in any programming language (not just Python) and is generally really easy to work with (you can execute arbitrary shell commands on events). You should choose this method unless you have good reason not to. This involves creating your YAML file (see `test_fsm.yml` for an example) and parsing it using our `YamlFSM` class (see `event_handler_main.py` for an example). @@ -224,6 +224,12 @@ Transition callbacks: - `before` - `after` +In your YAML file you can use these in the state and transition objects as keys, then add a shell command to run as a value (again, see `test_fsm.yml` for examples). + +It is also possible to add preconditions to transitions. +This is done by adding a `predicates` key with a list of shell commands to run. +If you do this, the transition will only succeed if the return code of all predicates was `0` (as per unix convention for success). + ## Baby steps When creating your own challenge the process should be the following: