mirror of
https://github.com/avatao-content/baseimage-tutorial-framework
synced 2024-11-21 17:11:32 +00:00
Merge branch 'ocicat', the unrealized dream. Ocicat will return...
This commit is contained in:
commit
07cd1264f5
@ -10,4 +10,4 @@ pipeline:
|
||||
- docker push eu.gcr.io/avatao-challengestore/tutorial-framework:${DRONE_TAG}
|
||||
when:
|
||||
event: 'tag'
|
||||
branch: refs/tags/mainecoon-20*
|
||||
branch: refs/tags/ocicat-20*
|
||||
|
22
.git-hooks/apply_hooks.sh
Executable file
22
.git-hooks/apply_hooks.sh
Executable file
@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
set -o pipefail
|
||||
set -o errtrace
|
||||
shopt -s expand_aliases
|
||||
|
||||
[ "$(uname)" == "Darwin" ] && alias readlink="greadlink" || :
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
|
||||
here="$(dirname "$(readlink -f "$0")")"
|
||||
cd "${here}/../.git/hooks"
|
||||
|
||||
rm -f pre-push pre-commit || :
|
||||
prepush_script="../../.git-hooks/pre-push.sh"
|
||||
precommit_script="../../.git-hooks/pre-commit.sh"
|
||||
[ -f "${prepush_script}" ] && ln -s "${prepush_script}" pre-push
|
||||
[ -f "${precommit_script}" ] && ln -s "${precommit_script}" pre-commit
|
||||
|
||||
echo -e "\n${GREEN}Done! Hooks applied, you can start committing and pushing!${NC}\n"
|
18
.git-hooks/pre-push.sh
Executable file
18
.git-hooks/pre-push.sh
Executable file
@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
set -o pipefail
|
||||
set -o errtrace
|
||||
shopt -s expand_aliases
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
|
||||
echo -e "Running pylint...\n"
|
||||
if pylint lib; then
|
||||
echo -e "\n${GREEN}Pylint found no errors!${NC}\n"
|
||||
else
|
||||
echo -e "\n${RED}Pylint failed with errors${NC}\n"
|
||||
exit 1
|
||||
fi
|
@ -3,3 +3,9 @@
|
||||
ignored-modules = zmq
|
||||
max-line-length = 120
|
||||
disable = missing-docstring, too-few-public-methods, invalid-name
|
||||
|
||||
[SIMILARITIES]
|
||||
|
||||
ignore-comments=yes
|
||||
ignore-docstrings=yes
|
||||
ignore-imports=yes
|
||||
|
@ -36,9 +36,10 @@ ENV PYTHONPATH="/usr/local/lib" \
|
||||
TFW_NGINX_DEFAULT="/etc/nginx/sites-enabled/default" \
|
||||
TFW_NGINX_COMPONENTS="/etc/nginx/components" \
|
||||
TFW_LIB_DIR="/usr/local/lib" \
|
||||
TFW_TERMINADO_DIR="/tmp/terminado_server" \
|
||||
TFW_FRONTEND_DIR="/srv/frontend" \
|
||||
TFW_SERVER_DIR="/srv/.tfw" \
|
||||
TFW_DIR="/.tfw" \
|
||||
TFW_SERVER_DIR="/.tfw/tfw_server" \
|
||||
TFW_SNAPSHOTS_DIR="/.tfw/snapshots" \
|
||||
TFW_AUTH_KEY="/tmp/tfw-auth.key" \
|
||||
TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history" \
|
||||
PROMPT_COMMAND="history -a"
|
||||
@ -75,4 +76,4 @@ ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn install --frozen-lockfil
|
||||
ONBUILD RUN test -z "${NOFRONTEND}" && cd /data && yarn build --no-progress || :
|
||||
ONBUILD RUN test -z "${NOFRONTEND}" && mv /data/dist ${TFW_FRONTEND_DIR} && rm -rf /data || :
|
||||
|
||||
CMD exec supervisord --nodaemon
|
||||
CMD exec supervisord --nodaemon --configuration ${TFW_SUPERVISORD_CONF}
|
||||
|
65
README.md
65
README.md
@ -47,6 +47,8 @@ Our pre-made event handlers are written in Python3, but you can write event hand
|
||||
This makes the framework really flexible: you can demonstrate the concepts you want to in any language while using the same set of tools provided by TFW.
|
||||
Inside Avatao this means that any of the content teams can use the framework with ease.
|
||||
|
||||
To implement an event handler in Python3 you should subclass the `EventHandlerBase` or `FSMAwareEventHandler` class in `tfw.event_handler_base` (the first provides a minimal working `EventHandler`, the second allows you to execute code on FSM events).
|
||||
|
||||
### FSM
|
||||
|
||||
Another unique feature of the framework is the FSM – finite state machine – representing the state of your challenge.
|
||||
@ -74,20 +76,24 @@ The TFW message format:
|
||||
|
||||
```text
|
||||
{
|
||||
"key: "some identifier used for addressing",
|
||||
"key: ...some identifier used for addressing...,
|
||||
"data":
|
||||
{
|
||||
...
|
||||
JSON object carrying anything, preferably cats
|
||||
...
|
||||
},
|
||||
"trigger": "FSM action"
|
||||
"trigger": ...FSM action...,
|
||||
"signature": ...HMAC signature for authenticated messages...,
|
||||
"seq": ...sequence number...
|
||||
}
|
||||
```
|
||||
|
||||
- The `key` field is used by TFW for addressing and every message must have one (it can be an empty string though)
|
||||
- The `data` object can contain anything you might want to send
|
||||
- The `trigger` key is an optional field that triggers an FSM action with that name from the current state (whatever that might be)
|
||||
- The `signature` field is present on authenticated messages (such as `fsm_update`s)
|
||||
- The `seq` key is a counter incremented with each proxied message in the TFW server
|
||||
|
||||
To mirror messages back to their sources you can use a special messaging format, in which the message to be mirrored is enveloped inside the `data` field of the outer message:
|
||||
|
||||
@ -117,6 +123,8 @@ APIs exposed by our pre-witten event handlers are documented here.
|
||||
|
||||
### IdeEventHandler
|
||||
|
||||
This event handler is responsible for reading and writing files shown in the frontend code editor.
|
||||
|
||||
You can read the content of the currently selected file like so:
|
||||
```
|
||||
{
|
||||
@ -178,6 +186,15 @@ Overwriting the current list of excluded file patterns is possible with this mes
|
||||
|
||||
### TerminalEventHandler
|
||||
|
||||
Event handler responsible for running a backend for `xterm.js` to connect to (frontend terminal backend).
|
||||
|
||||
By default callbacks on terminal history are invoked *as soon as* a command starts to execute in the terminal (they do not wait for the started command to finish, the callback may even run in paralell with the command).
|
||||
|
||||
If you want to wait for them and invoke your callbacks *after* the command has finished, please set the `TFW_DELAY_HISTAPPEND` envvar to `1`.
|
||||
Practically this can be done by appending an `export` to the user's `.bashrc` file from your `Dockerfile`, like so:
|
||||
|
||||
`RUN echo "export TFW_DELAY_HISTAPPEND=1" >> /home/${AVATAO_USER}/.bashrc`
|
||||
|
||||
Writing to the terminal:
|
||||
```
|
||||
{
|
||||
@ -204,6 +221,8 @@ You can read terminal command history like so:
|
||||
|
||||
### ProcessManagingEventHandler
|
||||
|
||||
This event handler is responsible for managing processes controlled by supervisord.
|
||||
|
||||
Starting, stopping and restarting supervisor processes can be done using similar messages (where `command` is `start`, `stop` or `restart`):
|
||||
```
|
||||
{
|
||||
@ -218,6 +237,8 @@ Starting, stopping and restarting supervisor processes can be done using similar
|
||||
|
||||
### LogMonitoringEventHandler
|
||||
|
||||
Event handler emitting real time logs (`stdout` and `stderr`) from supervisord processes.
|
||||
|
||||
To change which supervisor process is monitored use this message:
|
||||
```
|
||||
{
|
||||
@ -244,6 +265,8 @@ To set the tail length of logs (the monitor will send back the last `value` char
|
||||
|
||||
### FSMManagingEventHandler
|
||||
|
||||
This event handler controls the TFW finite state machine (FSM).
|
||||
|
||||
To attempt executing a trigger on the FSM use (this will also generate an FSM update message):
|
||||
```
|
||||
{
|
||||
@ -279,3 +302,41 @@ This event handler broadcasts FSM update messages after handling commands in the
|
||||
}
|
||||
```
|
||||
|
||||
### DirectorySnapshottingEventHandler
|
||||
|
||||
Event handler capable of taking and restoring snapshots of directories (saving and restoring directory contens).
|
||||
|
||||
You can take a snapshot of the directories with the following message:
|
||||
```
|
||||
{
|
||||
"key": "snapshot",
|
||||
"data" :
|
||||
{
|
||||
"command": "take_snapshot"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
To restore the state of the files in the directories use:
|
||||
```
|
||||
{
|
||||
"key": "snapshot",
|
||||
"data" :
|
||||
{
|
||||
"command": "restore_snapshot",
|
||||
"value": ...date string (can parse ISO 8601, unix timestamp, etc.)...
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
It is also possible to exclude files that match given patterns (formatted like lines in `.gitignore` files):
|
||||
```
|
||||
{
|
||||
"key": "snapshot",
|
||||
"data" :
|
||||
{
|
||||
"command": "exclude",
|
||||
"value": ...list of patterns to exclude from snapshots...
|
||||
}
|
||||
}
|
||||
```
|
||||
|
@ -23,3 +23,10 @@ Components
|
||||
|
||||
.. autoclass:: BashMonitor
|
||||
:members:
|
||||
|
||||
.. autoclass:: FSMManagingEventHandler
|
||||
:members:
|
||||
|
||||
.. autoclass:: CommandsEqual
|
||||
:members:
|
||||
|
||||
|
@ -3,10 +3,13 @@ Event handler base classes
|
||||
|
||||
Subclass these to create your cusom event handlers.
|
||||
|
||||
.. automodule:: tfw
|
||||
.. automodule:: tfw.event_handler_base
|
||||
|
||||
.. autoclass:: EventHandlerBase
|
||||
:members:
|
||||
|
||||
.. autoclass:: TriggeredEventHandler
|
||||
.. autoclass:: FSMAwareEventHandler
|
||||
:members:
|
||||
|
||||
.. autoclass:: BroadcastingEventHandler
|
||||
:members:
|
||||
|
@ -3,10 +3,13 @@ FSM base classes
|
||||
|
||||
Subclass these to create an FSM that fits your tutorial/challenge.
|
||||
|
||||
.. automodule:: tfw
|
||||
.. automodule:: tfw.fsm
|
||||
|
||||
.. autoclass:: FSMBase
|
||||
:members:
|
||||
|
||||
.. autoclass:: LinearFSM
|
||||
:members:
|
||||
|
||||
.. autoclass:: YamlFSM
|
||||
:members:
|
||||
|
@ -36,6 +36,16 @@ These are pre-written components for you to use, such as our IDE, terminal or co
|
||||
|
||||
components/*
|
||||
|
||||
Utility
|
||||
-------
|
||||
|
||||
These are useful decorators, mixins and helpers to make common dev tasks easier.
|
||||
|
||||
.. toctree::
|
||||
:glob:
|
||||
|
||||
utility/*
|
||||
|
||||
|
||||
Indices and tables
|
||||
==================
|
||||
|
@ -6,7 +6,7 @@ Networking
|
||||
.. autoclass:: TFWServerConnector
|
||||
:members:
|
||||
|
||||
.. automodule:: tfw.networking.event_handlers
|
||||
.. automodule:: tfw.networking.event_handlers.server_connector
|
||||
|
||||
.. autoclass:: ServerUplinkConnector
|
||||
:members:
|
||||
@ -15,3 +15,8 @@ Networking
|
||||
|
||||
.. autoclass:: MessageSender
|
||||
:members:
|
||||
|
||||
.. automodule:: tfw.networking.fsm_aware
|
||||
|
||||
.. autoclass:: FSMAware
|
||||
:members:
|
||||
|
10
docs/source/utility/decorators.rst
Normal file
10
docs/source/utility/decorators.rst
Normal file
@ -0,0 +1,10 @@
|
||||
TFW decorators
|
||||
--------------
|
||||
|
||||
.. automodule:: tfw.decorators.rate_limiter
|
||||
|
||||
.. autoclass:: RateLimiter
|
||||
:members:
|
||||
|
||||
.. autoclass:: AsyncRateLimiter
|
||||
:members:
|
@ -4,7 +4,7 @@
|
||||
from collections import namedtuple
|
||||
from os import environ
|
||||
|
||||
from tfw.decorators import lazy_property
|
||||
from tfw.decorators.lazy_property import lazy_property
|
||||
|
||||
|
||||
class LazyEnvironment:
|
||||
|
@ -1,7 +1,2 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from .event_handler_base import EventHandlerBase, FSMAwareEventHandler, BroadcastingEventHandler
|
||||
from .fsm_base import FSMBase
|
||||
from .linear_fsm import LinearFSM
|
||||
from .yaml_fsm import YamlFSM
|
||||
|
@ -12,3 +12,5 @@ from .fsm_managing_event_handler import FSMManagingEventHandler
|
||||
from .snapshot_provider import SnapshotProvider
|
||||
from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler, PipeIOServer
|
||||
from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler
|
||||
from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler
|
||||
from .commands_equal import CommandsEqual
|
||||
|
110
lib/tfw/components/commands_equal.py
Normal file
110
lib/tfw/components/commands_equal.py
Normal file
@ -0,0 +1,110 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from shlex import split
|
||||
from re import search
|
||||
|
||||
from tfw.decorators.lazy_property import lazy_property
|
||||
|
||||
|
||||
class CommandsEqual:
|
||||
# pylint: disable=too-many-arguments
|
||||
"""
|
||||
This class is useful for comparing executed commands with
|
||||
excepted commands (i.e. when triggering a state change when
|
||||
the correct command is executed).
|
||||
|
||||
Note that in most cases you should test the changes
|
||||
caused by the commands instead of just checking command history
|
||||
(stuff can be done in countless ways and preparing for every
|
||||
single case is impossible). This should only be used when
|
||||
testing the changes would be very difficult, like when
|
||||
explaining stuff with cli tools and such.
|
||||
|
||||
This class implicitly converts to bool, use it like
|
||||
if CommandsEqual(...): ...
|
||||
|
||||
It tries detecting differing command parameter orders with similar
|
||||
semantics and provides fuzzy logic options.
|
||||
The rationale behind this is that a few false positives
|
||||
are better than only accepting a single version of a command
|
||||
(i.e. using ==).
|
||||
"""
|
||||
def __init__(
|
||||
self, command_1, command_2,
|
||||
fuzzyness=1, begin_similarly=True,
|
||||
include_patterns=None, exclude_patterns=None
|
||||
):
|
||||
"""
|
||||
:param command_1: Compared command 1
|
||||
:param command_2: Compared command 2
|
||||
:param fuzzyness: float between 0 and 1.
|
||||
the percentage of arguments required to
|
||||
match between commands to result in True.
|
||||
i.e 1 means 100% - all arguments need to be
|
||||
present in both commands, while 0.75
|
||||
would mean 75% - in case of 4 arguments
|
||||
1 could differ between the commands.
|
||||
:param begin_similarly: bool, the first word of the commands
|
||||
must match
|
||||
:param include_patterns: list of regex patterns the commands
|
||||
must include
|
||||
:param exclude_patterns: list of regex patterns the commands
|
||||
must exclude
|
||||
"""
|
||||
self.command_1 = split(command_1)
|
||||
self.command_2 = split(command_2)
|
||||
self.fuzzyness = fuzzyness
|
||||
self.begin_similarly = begin_similarly
|
||||
self.include_patterns = include_patterns
|
||||
self.exclude_patterns = exclude_patterns
|
||||
|
||||
def __bool__(self):
|
||||
if self.begin_similarly:
|
||||
if not self.beginnings_are_equal:
|
||||
return False
|
||||
|
||||
if self.include_patterns is not None:
|
||||
if not self.commands_contain_include_patterns:
|
||||
return False
|
||||
|
||||
if self.exclude_patterns is not None:
|
||||
if not self.commands_contain_no_exclude_patterns:
|
||||
return False
|
||||
|
||||
return self.similarity >= self.fuzzyness
|
||||
|
||||
@lazy_property
|
||||
def beginnings_are_equal(self):
|
||||
return self.command_1[0] == self.command_2[0]
|
||||
|
||||
@lazy_property
|
||||
def commands_contain_include_patterns(self):
|
||||
return all((
|
||||
self.contains_regex_patterns(self.command_1, self.include_patterns),
|
||||
self.contains_regex_patterns(self.command_2, self.include_patterns)
|
||||
))
|
||||
|
||||
@lazy_property
|
||||
def commands_contain_no_exclude_patterns(self):
|
||||
return all((
|
||||
not self.contains_regex_patterns(self.command_1, self.exclude_patterns),
|
||||
not self.contains_regex_patterns(self.command_2, self.exclude_patterns)
|
||||
))
|
||||
|
||||
@staticmethod
|
||||
def contains_regex_patterns(command, regex_parts):
|
||||
command = ' '.join(command)
|
||||
for pattern in regex_parts:
|
||||
if not search(pattern, command):
|
||||
return False
|
||||
return True
|
||||
|
||||
@lazy_property
|
||||
def similarity(self):
|
||||
parts_1 = set(self.command_1)
|
||||
parts_2 = set(self.command_2)
|
||||
|
||||
difference = parts_1 - parts_2
|
||||
deviance = len(difference) / len(max(parts_1, parts_2))
|
||||
return 1 - deviance
|
@ -5,9 +5,9 @@ from functools import wraps
|
||||
|
||||
from watchdog.events import FileSystemEventHandler as FileSystemWatchdogEventHandler
|
||||
|
||||
from tfw.networking.event_handlers import ServerUplinkConnector
|
||||
from tfw.decorators import RateLimiter
|
||||
from tfw.mixins import ObserverMixin
|
||||
from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
|
||||
from tfw.decorators.rate_limiter import RateLimiter
|
||||
from tfw.mixins.observer_mixin import ObserverMixin
|
||||
|
||||
from tfw.config.logs import logging
|
||||
|
||||
|
@ -3,10 +3,10 @@
|
||||
|
||||
from os.path import isdir, exists
|
||||
|
||||
from tfw import EventHandlerBase
|
||||
from tfw.event_handler_base import EventHandlerBase
|
||||
from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin
|
||||
from tfw.components.directory_monitor import DirectoryMonitor
|
||||
from tfw.config.logs import logging
|
||||
from tfw.mixins import MonitorManagerMixin
|
||||
from .directory_monitor import DirectoryMonitor
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
87
lib/tfw/components/directory_snapshotting_event_handler.py
Normal file
87
lib/tfw/components/directory_snapshotting_event_handler.py
Normal file
@ -0,0 +1,87 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from os.path import join as joinpath
|
||||
from os.path import basename
|
||||
from os import makedirs
|
||||
from datetime import datetime
|
||||
|
||||
from dateutil import parser as dateparser
|
||||
|
||||
from tfw.event_handler_base import EventHandlerBase
|
||||
from tfw.components.snapshot_provider import SnapshotProvider
|
||||
from tfw.config import TFWENV
|
||||
from tfw.config.logs import logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DirectorySnapshottingEventHandler(EventHandlerBase):
|
||||
def __init__(self, key, directories, exclude_unix_patterns=None):
|
||||
super().__init__(key)
|
||||
self.snapshot_providers = {}
|
||||
self._exclude_unix_patterns = exclude_unix_patterns
|
||||
self.init_snapshot_providers(directories)
|
||||
|
||||
self.command_handlers = {
|
||||
'take_snapshot': self.handle_take_snapshot,
|
||||
'restore_snapshot': self.handle_restore_snapshot,
|
||||
'exclude': self.handle_exclude
|
||||
}
|
||||
|
||||
def init_snapshot_providers(self, directories):
|
||||
for index, directory in enumerate(directories):
|
||||
git_dir = self.init_git_dir(index, directory)
|
||||
self.snapshot_providers[directory] = SnapshotProvider(
|
||||
directory,
|
||||
git_dir,
|
||||
self._exclude_unix_patterns
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def init_git_dir(index, directory):
|
||||
git_dir = joinpath(
|
||||
TFWENV.SNAPSHOTS_DIR,
|
||||
f'{basename(directory)}-{index}'
|
||||
)
|
||||
makedirs(git_dir, exist_ok=True)
|
||||
return git_dir
|
||||
|
||||
def handle_event(self, message):
|
||||
try:
|
||||
data = message['data']
|
||||
message['data'] = self.command_handlers[data['command']](data)
|
||||
return message
|
||||
except KeyError:
|
||||
LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
|
||||
|
||||
def handle_take_snapshot(self, data):
|
||||
LOG.debug('Taking snapshots of directories %s', self.snapshot_providers.keys())
|
||||
for provider in self.snapshot_providers.values():
|
||||
provider.take_snapshot()
|
||||
return data
|
||||
|
||||
def handle_restore_snapshot(self, data):
|
||||
date = dateparser.parse(
|
||||
data.get(
|
||||
'value',
|
||||
datetime.now().isoformat()
|
||||
)
|
||||
)
|
||||
LOG.debug(
|
||||
'Restoring snapshots (@ %s) of directories %s',
|
||||
date,
|
||||
self.snapshot_providers.keys()
|
||||
)
|
||||
for provider in self.snapshot_providers.values():
|
||||
provider.restore_snapshot(date)
|
||||
return data
|
||||
|
||||
def handle_exclude(self, data):
|
||||
exclude_unix_patterns = data['value']
|
||||
if not isinstance(exclude_unix_patterns, list):
|
||||
raise KeyError
|
||||
|
||||
for provider in self.snapshot_providers.values():
|
||||
provider.exclude = exclude_unix_patterns
|
||||
return data
|
@ -1,7 +1,7 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from tfw import EventHandlerBase
|
||||
from tfw.event_handler_base import EventHandlerBase
|
||||
from tfw.crypto import KeyManager, sign_message, verify_message
|
||||
from tfw.config.logs import logging
|
||||
|
||||
@ -9,6 +9,20 @@ LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FSMManagingEventHandler(EventHandlerBase):
|
||||
"""
|
||||
EventHandler responsible for managing the state machine of
|
||||
the framework (TFW FSM).
|
||||
|
||||
tfw.networking.TFWServer instances automatically send 'trigger'
|
||||
commands to the event handler listening on the 'fsm' key,
|
||||
which should be an instance of this event handler.
|
||||
|
||||
This event handler accepts messages that have a
|
||||
data['command'] key specifying a command to be executed.
|
||||
|
||||
An 'fsm_update' message is broadcasted after every successful
|
||||
command.
|
||||
"""
|
||||
def __init__(self, key, fsm_type, require_signature=False):
|
||||
super().__init__(key)
|
||||
self.fsm = fsm_type()
|
||||
@ -25,7 +39,7 @@ class FSMManagingEventHandler(EventHandlerBase):
|
||||
try:
|
||||
message = self.command_handlers[message['data']['command']](message)
|
||||
if message:
|
||||
fsm_update_message = self._fsm_updater.generate_fsm_update()
|
||||
fsm_update_message = self._fsm_updater.fsm_update
|
||||
sign_message(self.auth_key, message)
|
||||
sign_message(self.auth_key, fsm_update_message)
|
||||
self.server_connector.broadcast(fsm_update_message)
|
||||
@ -34,6 +48,12 @@ class FSMManagingEventHandler(EventHandlerBase):
|
||||
LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
|
||||
|
||||
def handle_trigger(self, message):
|
||||
"""
|
||||
Attempts to step the FSM with the supplied trigger.
|
||||
|
||||
:param message: TFW message with a data field containing
|
||||
the action to try triggering in data['value']
|
||||
"""
|
||||
trigger = message['data']['value']
|
||||
if self._require_signature:
|
||||
if not verify_message(self.auth_key, message):
|
||||
@ -44,6 +64,9 @@ class FSMManagingEventHandler(EventHandlerBase):
|
||||
return None
|
||||
|
||||
def handle_update(self, message):
|
||||
"""
|
||||
Does nothing, but triggers an 'fsm_update' message.
|
||||
"""
|
||||
# pylint: disable=no-self-use
|
||||
return message
|
||||
|
||||
@ -52,23 +75,24 @@ class FSMUpdater:
|
||||
def __init__(self, fsm):
|
||||
self.fsm = fsm
|
||||
|
||||
def generate_fsm_update(self):
|
||||
@property
|
||||
def fsm_update(self):
|
||||
return {
|
||||
'key': 'fsm_update',
|
||||
'data': self.get_fsm_state_and_transitions()
|
||||
'data': self.fsm_update_data
|
||||
}
|
||||
|
||||
def get_fsm_state_and_transitions(self):
|
||||
state = self.fsm.state
|
||||
@property
|
||||
def fsm_update_data(self):
|
||||
valid_transitions = [
|
||||
{'trigger': trigger}
|
||||
for trigger in self.fsm.get_triggers(self.fsm.state)
|
||||
]
|
||||
last_trigger = self.fsm.trigger_history[-1] if self.fsm.trigger_history else None
|
||||
in_accepted_state = state in self.fsm.accepted_states
|
||||
last_fsm_event = self.fsm.event_log[-1]
|
||||
last_fsm_event['timestamp'] = last_fsm_event['timestamp'].isoformat()
|
||||
return {
|
||||
'current_state': state,
|
||||
'current_state': self.fsm.state,
|
||||
'valid_transitions': valid_transitions,
|
||||
'last_trigger': last_trigger,
|
||||
'in_accepted_state': in_accepted_state
|
||||
'in_accepted_state': self.fsm.in_accepted_state,
|
||||
'last_event': last_fsm_event
|
||||
}
|
||||
|
@ -8,8 +8,9 @@ from abc import ABC, abstractmethod
|
||||
|
||||
from watchdog.events import PatternMatchingEventHandler
|
||||
|
||||
from tfw.mixins import CallbackMixin, ObserverMixin
|
||||
from tfw.decorators import RateLimiter
|
||||
from tfw.mixins.callback_mixin import CallbackMixin
|
||||
from tfw.mixins.observer_mixin import ObserverMixin
|
||||
from tfw.decorators.rate_limiter import RateLimiter
|
||||
|
||||
|
||||
class CallbackEventHandler(PatternMatchingEventHandler, ABC):
|
||||
|
@ -6,10 +6,10 @@ from glob import glob
|
||||
from fnmatch import fnmatchcase
|
||||
from typing import Iterable
|
||||
|
||||
from tfw import EventHandlerBase
|
||||
from tfw.mixins import MonitorManagerMixin
|
||||
from tfw.event_handler_base import EventHandlerBase
|
||||
from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin
|
||||
from tfw.components.directory_monitor import DirectoryMonitor
|
||||
from tfw.config.logs import logging
|
||||
from .directory_monitor import DirectoryMonitor
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
@ -157,7 +157,8 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
|
||||
"""
|
||||
Read the currently selected file.
|
||||
|
||||
:return dict: message with the contents of the file in data['content']
|
||||
:return dict: TFW message data containing key 'content'
|
||||
(contents of the selected file)
|
||||
"""
|
||||
try:
|
||||
data['content'] = self.filemanager.file_contents
|
||||
|
@ -6,9 +6,10 @@ from os.path import dirname
|
||||
|
||||
from watchdog.events import PatternMatchingEventHandler as PatternMatchingWatchdogEventHandler
|
||||
|
||||
from tfw.networking.event_handlers import ServerUplinkConnector
|
||||
from tfw.decorators import RateLimiter
|
||||
from tfw.mixins import ObserverMixin, SupervisorLogMixin
|
||||
from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
|
||||
from tfw.decorators.rate_limiter import RateLimiter
|
||||
from tfw.mixins.observer_mixin import ObserverMixin
|
||||
from tfw.mixins.supervisor_mixin import SupervisorLogMixin
|
||||
|
||||
|
||||
class LogMonitor(ObserverMixin):
|
||||
|
@ -1,10 +1,10 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from tfw import EventHandlerBase
|
||||
from tfw.mixins import MonitorManagerMixin
|
||||
from tfw.event_handler_base import EventHandlerBase
|
||||
from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin
|
||||
from tfw.components.log_monitor import LogMonitor
|
||||
from tfw.config.logs import logging
|
||||
from .log_monitor import LogMonitor
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
@ -9,7 +9,7 @@ from secrets import token_urlsafe
|
||||
from threading import Thread
|
||||
from contextlib import suppress
|
||||
|
||||
from tfw import EventHandlerBase
|
||||
from tfw.event_handler_base import EventHandlerBase
|
||||
from tfw.config.logs import logging
|
||||
|
||||
from .pipe_io_server import PipeIOServer, terminate_process_on_failure
|
||||
|
@ -3,10 +3,10 @@
|
||||
|
||||
from xmlrpc.client import Fault as SupervisorFault
|
||||
|
||||
from tfw import EventHandlerBase
|
||||
from tfw.mixins import SupervisorMixin, SupervisorLogMixin
|
||||
from tfw.event_handler_base import EventHandlerBase
|
||||
from tfw.mixins.supervisor_mixin import SupervisorMixin, SupervisorLogMixin
|
||||
from tfw.components.directory_monitor import with_monitor_paused
|
||||
from tfw.config.logs import logging
|
||||
from .directory_monitor import with_monitor_paused
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
@ -2,15 +2,17 @@
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
import re
|
||||
from subprocess import run, CalledProcessError
|
||||
from subprocess import run, CalledProcessError, PIPE
|
||||
from getpass import getuser
|
||||
from os.path import isdir
|
||||
from datetime import datetime
|
||||
from os.path import join as joinpath
|
||||
from uuid import uuid4
|
||||
|
||||
from dateutil import parser as dateparser
|
||||
|
||||
|
||||
class SnapshotProvider:
|
||||
def __init__(self, directory, git_dir):
|
||||
def __init__(self, directory, git_dir, exclude_unix_patterns=None):
|
||||
self._classname = self.__class__.__name__
|
||||
author = f'{getuser()} via TFW {self._classname}'
|
||||
self.gitenv = {
|
||||
@ -25,6 +27,8 @@ class SnapshotProvider:
|
||||
|
||||
self._init_repo()
|
||||
self.__last_valid_branch = self._branch
|
||||
if exclude_unix_patterns:
|
||||
self.exclude = exclude_unix_patterns
|
||||
|
||||
def _init_repo(self):
|
||||
self._check_environment()
|
||||
@ -66,10 +70,14 @@ class SnapshotProvider:
|
||||
'git', 'add',
|
||||
'-A'
|
||||
))
|
||||
self._run((
|
||||
'git', 'commit',
|
||||
'-m', 'Snapshot'
|
||||
))
|
||||
try:
|
||||
self._get_stdout((
|
||||
'git', 'commit',
|
||||
'-m', 'Snapshot'
|
||||
))
|
||||
except CalledProcessError as err:
|
||||
if b'nothing to commit, working tree clean' not in err.output:
|
||||
raise
|
||||
|
||||
def _check_head_not_detached(self):
|
||||
if self._head_detached:
|
||||
@ -87,7 +95,8 @@ class SnapshotProvider:
|
||||
))
|
||||
|
||||
def _get_stdout(self, *args, **kwargs):
|
||||
kwargs['capture_output'] = True
|
||||
kwargs['stdout'] = PIPE
|
||||
kwargs['stderr'] = PIPE
|
||||
stdout_bytes = self._run(*args, **kwargs).stdout
|
||||
return stdout_bytes.decode().rstrip('\n')
|
||||
|
||||
@ -98,13 +107,31 @@ class SnapshotProvider:
|
||||
kwargs['env'] = self.gitenv
|
||||
return run(*args, **kwargs)
|
||||
|
||||
@property
|
||||
def exclude(self):
|
||||
with open(self._exclude_path, 'r') as ofile:
|
||||
return ofile.read()
|
||||
|
||||
@exclude.setter
|
||||
def exclude(self, exclude_patterns):
|
||||
with open(self._exclude_path, 'w') as ifile:
|
||||
ifile.write('\n'.join(exclude_patterns))
|
||||
|
||||
@property
|
||||
def _exclude_path(self):
|
||||
return joinpath(
|
||||
self.gitenv['GIT_DIR'],
|
||||
'info',
|
||||
'exclude'
|
||||
)
|
||||
|
||||
def take_snapshot(self):
|
||||
if self._head_detached:
|
||||
self._checkout_new_branch_from_head()
|
||||
self._snapshot()
|
||||
|
||||
def _checkout_new_branch_from_head(self):
|
||||
branch_name = uuid4()
|
||||
branch_name = str(uuid4())
|
||||
self._run((
|
||||
'git', 'branch',
|
||||
branch_name
|
||||
@ -119,16 +146,30 @@ class SnapshotProvider:
|
||||
|
||||
def restore_snapshot(self, date):
|
||||
commit = self._get_commit_from_timestamp(date)
|
||||
branch = self._last_valid_branch
|
||||
if commit == self._latest_commit_on_branch(branch):
|
||||
commit = branch
|
||||
self._checkout(commit)
|
||||
|
||||
def _get_commit_from_timestamp(self, date):
|
||||
return self._get_stdout((
|
||||
commit = self._get_stdout((
|
||||
'git', 'rev-list',
|
||||
'--date=iso',
|
||||
'-n', '1',
|
||||
f'--before="{date.isoformat()}"',
|
||||
self._last_valid_branch
|
||||
))
|
||||
if not commit:
|
||||
commit = self._get_oldest_parent_of_head()
|
||||
return commit
|
||||
|
||||
def _get_oldest_parent_of_head(self):
|
||||
return self._get_stdout((
|
||||
'git',
|
||||
'rev-list',
|
||||
'--max-parents=0',
|
||||
'HEAD'
|
||||
))
|
||||
|
||||
@property
|
||||
def _last_valid_branch(self):
|
||||
@ -136,6 +177,14 @@ class SnapshotProvider:
|
||||
self.__last_valid_branch = self._branch
|
||||
return self.__last_valid_branch
|
||||
|
||||
def _latest_commit_on_branch(self, branch):
|
||||
return self._get_stdout((
|
||||
'git', 'log',
|
||||
'-n', '1',
|
||||
'--pretty=format:%H',
|
||||
branch
|
||||
))
|
||||
|
||||
@property
|
||||
def all_timelines(self):
|
||||
return self._branches
|
||||
@ -169,7 +218,7 @@ class SnapshotProvider:
|
||||
commit_hash, timestamp = line.split('@')
|
||||
commits.append({
|
||||
'hash': commit_hash,
|
||||
'timestamp': datetime.fromisoformat(timestamp)
|
||||
'timestamp': dateparser.parse(timestamp)
|
||||
})
|
||||
|
||||
return commits
|
||||
|
@ -1,11 +1,11 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from tfw import EventHandlerBase
|
||||
from tfw.event_handler_base import EventHandlerBase
|
||||
from tfw.components.terminado_mini_server import TerminadoMiniServer
|
||||
from tfw.config import TFWENV
|
||||
from tfw.config.logs import logging
|
||||
from tao.config import TAOENV
|
||||
from .terminado_mini_server import TerminadoMiniServer
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
@ -26,7 +26,6 @@ class TerminalEventHandler(EventHandlerBase):
|
||||
:param monitor: tfw.components.HistoryMonitor instance to read command history from
|
||||
"""
|
||||
super().__init__(key)
|
||||
self.working_directory = TFWENV.TERMINADO_DIR
|
||||
self._historymonitor = monitor
|
||||
bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash']
|
||||
|
||||
|
@ -14,8 +14,8 @@ from cryptography.hazmat.primitives.hashes import SHA256
|
||||
from cryptography.hazmat.primitives.hmac import HMAC as _HMAC
|
||||
from cryptography.exceptions import InvalidSignature
|
||||
|
||||
from tfw.networking import message_bytes
|
||||
from tfw.decorators import lazy_property
|
||||
from tfw.networking.serialization import message_bytes
|
||||
from tfw.decorators.lazy_property import lazy_property
|
||||
from tfw.config import TFWENV
|
||||
|
||||
|
||||
|
@ -1,5 +1,2 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from .rate_limiter import RateLimiter
|
||||
from .lazy_property import lazy_property
|
||||
|
@ -1,7 +1,7 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from functools import update_wrapper
|
||||
from functools import update_wrapper, wraps
|
||||
|
||||
|
||||
class lazy_property:
|
||||
@ -19,3 +19,12 @@ class lazy_property:
|
||||
value = self.func(instance)
|
||||
setattr(instance, self.func.__name__, value)
|
||||
return value
|
||||
|
||||
|
||||
def lazy_factory(fun):
|
||||
class wrapper:
|
||||
@wraps(fun)
|
||||
@lazy_property
|
||||
def instance(self): # pylint: disable=no-self-use
|
||||
return fun()
|
||||
return wrapper()
|
||||
|
@ -87,6 +87,7 @@ class AsyncRateLimiter(RateLimiter):
|
||||
return self._ioloop_factory()
|
||||
|
||||
def action(self, seconds_to_next_call):
|
||||
# pylint: disable=method-hidden
|
||||
if self._last_callback:
|
||||
self.ioloop.remove_timeout(self._last_callback)
|
||||
|
||||
|
6
lib/tfw/event_handler_base/__init__.py
Normal file
6
lib/tfw/event_handler_base/__init__.py
Normal file
@ -0,0 +1,6 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from .event_handler_base import EventHandlerBase
|
||||
from .boradcasting_event_handler import BroadcastingEventHandler
|
||||
from .fsm_aware_event_handler import FSMAwareEventHandler
|
30
lib/tfw/event_handler_base/boradcasting_event_handler.py
Normal file
30
lib/tfw/event_handler_base/boradcasting_event_handler.py
Normal file
@ -0,0 +1,30 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from abc import ABC
|
||||
|
||||
from tfw.event_handler_base.event_handler_base import EventHandlerBase
|
||||
from tfw.crypto import message_checksum
|
||||
|
||||
|
||||
class BroadcastingEventHandler(EventHandlerBase, ABC):
|
||||
# pylint: disable=abstract-method
|
||||
"""
|
||||
Abstract base class for EventHandlers which broadcast responses
|
||||
and intelligently ignore their own broadcasted messages they receive.
|
||||
"""
|
||||
def __init__(self, key):
|
||||
super().__init__(key)
|
||||
self.own_message_hashes = []
|
||||
|
||||
def event_handler_callback(self, message):
|
||||
message_hash = message_checksum(message)
|
||||
|
||||
if message_hash in self.own_message_hashes:
|
||||
self.own_message_hashes.remove(message_hash)
|
||||
return
|
||||
|
||||
response = self.dispatch_handling(message)
|
||||
if response:
|
||||
self.own_message_hashes.append(message_checksum(response))
|
||||
self.server_connector.broadcast(response)
|
@ -5,8 +5,7 @@ from abc import ABC, abstractmethod
|
||||
from inspect import currentframe
|
||||
from typing import Iterable
|
||||
|
||||
from tfw.networking.event_handlers import ServerConnector
|
||||
from tfw.crypto import message_checksum, KeyManager, verify_message
|
||||
from tfw.networking.event_handlers.server_connector import ServerConnector
|
||||
from tfw.config.logs import logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
@ -125,64 +124,3 @@ class EventHandlerBase(ABC):
|
||||
instance for instance in locals_values
|
||||
if isinstance(instance, cls)
|
||||
}
|
||||
|
||||
|
||||
class FSMAwareEventHandler(EventHandlerBase, ABC):
|
||||
# pylint: disable=abstract-method
|
||||
"""
|
||||
Abstract base class for EventHandlers which automatically
|
||||
keep track of the state of the TFW FSM.
|
||||
"""
|
||||
def __init__(self, key):
|
||||
super().__init__(key)
|
||||
self.subscribe('fsm_update')
|
||||
self.fsm_state = None
|
||||
self.in_accepted_state = False
|
||||
self._auth_key = KeyManager().auth_key
|
||||
|
||||
def dispatch_handling(self, message):
|
||||
if message['key'] == 'fsm_update':
|
||||
if verify_message(self._auth_key, message):
|
||||
self._handle_fsm_update(message)
|
||||
return None
|
||||
return super().dispatch_handling(message)
|
||||
|
||||
def _handle_fsm_update(self, message):
|
||||
try:
|
||||
new_state = message['data']['current_state']
|
||||
trigger = message['data']['last_trigger']
|
||||
if self.fsm_state != new_state:
|
||||
self.handle_fsm_step(self.fsm_state, new_state, trigger)
|
||||
self.fsm_state = new_state
|
||||
self.in_accepted_state = message['data']['in_accepted_state']
|
||||
except KeyError:
|
||||
LOG.error('Invalid fsm_update message received!')
|
||||
|
||||
def handle_fsm_step(self, from_state, to_state, trigger):
|
||||
"""
|
||||
Called in case the TFW FSM has stepped.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class BroadcastingEventHandler(EventHandlerBase, ABC):
|
||||
# pylint: disable=abstract-method
|
||||
"""
|
||||
Abstract base class for EventHandlers which broadcast responses
|
||||
and intelligently ignore their own broadcasted messages they receive.
|
||||
"""
|
||||
def __init__(self, key):
|
||||
super().__init__(key)
|
||||
self.own_message_hashes = []
|
||||
|
||||
def event_handler_callback(self, message):
|
||||
message_hash = message_checksum(message)
|
||||
|
||||
if message_hash in self.own_message_hashes:
|
||||
self.own_message_hashes.remove(message_hash)
|
||||
return
|
||||
|
||||
response = self.dispatch_handling(message)
|
||||
if response:
|
||||
self.own_message_hashes.append(message_checksum(response))
|
||||
self.server_connector.broadcast(response)
|
24
lib/tfw/event_handler_base/fsm_aware_event_handler.py
Normal file
24
lib/tfw/event_handler_base/fsm_aware_event_handler.py
Normal file
@ -0,0 +1,24 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from abc import ABC
|
||||
|
||||
from tfw.event_handler_base.event_handler_base import EventHandlerBase
|
||||
from tfw.networking.fsm_aware import FSMAware
|
||||
|
||||
|
||||
class FSMAwareEventHandler(EventHandlerBase, FSMAware, ABC):
|
||||
# pylint: disable=abstract-method
|
||||
"""
|
||||
Abstract base class for EventHandlers which automatically
|
||||
keep track of the state of the TFW FSM.
|
||||
"""
|
||||
def __init__(self, key):
|
||||
EventHandlerBase.__init__(self, key)
|
||||
FSMAware.__init__(self)
|
||||
self.subscribe('fsm_update')
|
||||
|
||||
def dispatch_handling(self, message):
|
||||
if self.update_fsm_data(message):
|
||||
return None
|
||||
return super().dispatch_handling(message)
|
6
lib/tfw/fsm/__init__.py
Normal file
6
lib/tfw/fsm/__init__.py
Normal file
@ -0,0 +1,6 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from .fsm_base import FSMBase
|
||||
from .linear_fsm import LinearFSM
|
||||
from .yaml_fsm import YamlFSM
|
@ -2,10 +2,11 @@
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
|
||||
from transitions import Machine, MachineError
|
||||
|
||||
from tfw.mixins import CallbackMixin
|
||||
from tfw.mixins.callback_mixin import CallbackMixin
|
||||
from tfw.config.logs import logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
@ -22,9 +23,14 @@ class FSMBase(Machine, CallbackMixin):
|
||||
states, transitions = [], []
|
||||
|
||||
def __init__(self, initial=None, accepted_states=None):
|
||||
"""
|
||||
:param initial: which state to begin with, defaults to the last one
|
||||
:param accepted_states: list of states in which the challenge should be
|
||||
considered successfully completed
|
||||
"""
|
||||
self.accepted_states = accepted_states or [self.states[-1].name]
|
||||
self.trigger_predicates = defaultdict(list)
|
||||
self.trigger_history = []
|
||||
self.event_log = []
|
||||
|
||||
Machine.__init__(
|
||||
self,
|
||||
@ -60,9 +66,22 @@ class FSMBase(Machine, CallbackMixin):
|
||||
|
||||
if all(predicate_results):
|
||||
try:
|
||||
from_state = self.state
|
||||
self.trigger(trigger)
|
||||
self.trigger_history.append(trigger)
|
||||
self.update_event_log(from_state, trigger)
|
||||
return True
|
||||
except (AttributeError, MachineError):
|
||||
LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
|
||||
return False
|
||||
|
||||
def update_event_log(self, from_state, trigger):
|
||||
self.event_log.append({
|
||||
'from_state': from_state,
|
||||
'to_state': self.state,
|
||||
'trigger': trigger,
|
||||
'timestamp': datetime.utcnow()
|
||||
})
|
||||
|
||||
@property
|
||||
def in_accepted_state(self):
|
||||
return self.state in self.accepted_states
|
@ -3,7 +3,7 @@
|
||||
|
||||
from transitions import State
|
||||
|
||||
from .fsm_base import FSMBase
|
||||
from tfw.fsm.fsm_base import FSMBase
|
||||
|
||||
|
||||
class LinearFSM(FSMBase):
|
||||
@ -12,9 +12,13 @@ class LinearFSM(FSMBase):
|
||||
This is a state machine for challenges with linear progression, consisting of
|
||||
a number of steps specified in the constructor. It automatically sets up 2
|
||||
actions (triggers) between states as such:
|
||||
(0) -- step_1 --> (1) -- step_2 --> (2) -- step_3 --> (3) ... and so on
|
||||
(0) -- step_1 --> (1) -- step_2 --> (2) -- step_3 --> (3) ...
|
||||
(0) -- step_next --> (1) -- step_next --> (2) -- step_next --> (3) ...
|
||||
"""
|
||||
def __init__(self, number_of_steps):
|
||||
"""
|
||||
:param number_of_steps: how many states this FSM should have
|
||||
"""
|
||||
self.states = [State(name=str(index)) for index in range(number_of_steps)]
|
||||
self.transitions = []
|
||||
for state in self.states[:-1]:
|
@ -1,3 +1,6 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from subprocess import Popen, run
|
||||
from functools import partial, singledispatch
|
||||
from contextlib import suppress
|
||||
@ -6,11 +9,21 @@ import yaml
|
||||
import jinja2
|
||||
from transitions import State
|
||||
|
||||
from tfw import FSMBase
|
||||
from tfw.fsm.fsm_base import FSMBase
|
||||
|
||||
|
||||
class YamlFSM(FSMBase):
|
||||
"""
|
||||
This is a state machine capable of building itself from a YAML config file.
|
||||
"""
|
||||
def __init__(self, config_file, jinja2_variables=None):
|
||||
"""
|
||||
:param config_file: path of the YAML file
|
||||
:param jinja2_variables: dict containing jinja2 variables
|
||||
or str with filename of YAML file to
|
||||
parse and use as dict.
|
||||
jinja2 support is disabled if this is None
|
||||
"""
|
||||
self.config = ConfigParser(config_file, jinja2_variables).config
|
||||
self.setup_states()
|
||||
super().__init__() # FSMBase.__init__() requires states
|
||||
@ -45,7 +58,7 @@ class YamlFSM(FSMBase):
|
||||
partial(
|
||||
command_statuscode_is_zero,
|
||||
predicate
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
with suppress(KeyError):
|
@ -1,7 +1,2 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from .supervisor_mixin import SupervisorMixin, SupervisorLogMixin
|
||||
from .callback_mixin import CallbackMixin
|
||||
from .observer_mixin import ObserverMixin
|
||||
from .monitor_manager_mixin import MonitorManagerMixin
|
||||
|
@ -3,7 +3,7 @@
|
||||
|
||||
from functools import partial
|
||||
|
||||
from tfw.decorators import lazy_property
|
||||
from tfw.decorators.lazy_property import lazy_property
|
||||
|
||||
|
||||
class CallbackMixin:
|
||||
|
@ -3,7 +3,7 @@
|
||||
|
||||
from watchdog.observers import Observer
|
||||
|
||||
from tfw.decorators import lazy_property
|
||||
from tfw.decorators.lazy_property import lazy_property
|
||||
|
||||
|
||||
class ObserverMixin:
|
||||
|
@ -6,7 +6,7 @@ from xmlrpc.client import Fault as SupervisorFault
|
||||
from contextlib import suppress
|
||||
from os import remove
|
||||
|
||||
from tfw.decorators import lazy_property
|
||||
from tfw.decorators.lazy_property import lazy_property
|
||||
from tfw.config import TFWENV
|
||||
|
||||
|
||||
|
@ -1,9 +1,6 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from .serialization import serialize_tfw_msg, deserialize_tfw_msg
|
||||
from .serialization import with_deserialize_tfw_msg, message_bytes
|
||||
from .zmq_connector_base import ZMQConnectorBase
|
||||
from .message_sender import MessageSender
|
||||
from .event_handlers.server_connector import ServerUplinkConnector as TFWServerConnector
|
||||
from .server.tfw_server import TFWServer
|
||||
|
@ -1,4 +1,2 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from .server_connector import ServerConnector, ServerUplinkConnector, ServerDownlinkConnector
|
||||
|
@ -6,8 +6,8 @@ from functools import partial
|
||||
import zmq
|
||||
from zmq.eventloop.zmqstream import ZMQStream
|
||||
|
||||
from tfw.networking import serialize_tfw_msg, with_deserialize_tfw_msg
|
||||
from tfw.networking import ZMQConnectorBase
|
||||
from tfw.networking.zmq_connector_base import ZMQConnectorBase
|
||||
from tfw.networking.serialization import serialize_tfw_msg, with_deserialize_tfw_msg
|
||||
from tfw.config import TFWENV
|
||||
from tfw.config.logs import logging
|
||||
|
||||
|
46
lib/tfw/networking/fsm_aware.py
Normal file
46
lib/tfw/networking/fsm_aware.py
Normal file
@ -0,0 +1,46 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from tfw.crypto import KeyManager, verify_message
|
||||
|
||||
from tfw.config.logs import logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FSMAware:
|
||||
"""
|
||||
Base class for stuff that has to be aware of the framework FSM.
|
||||
This is done by processing 'fsm_update' messages.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.fsm_state = None
|
||||
self.fsm_in_accepted_state = False
|
||||
self.fsm_event_log = []
|
||||
self._auth_key = KeyManager().auth_key
|
||||
|
||||
def update_fsm_data(self, message):
|
||||
if message['key'] == 'fsm_update' and verify_message(self._auth_key, message):
|
||||
self._handle_fsm_update(message)
|
||||
return True
|
||||
return False
|
||||
|
||||
def _handle_fsm_update(self, message):
|
||||
try:
|
||||
update_data = message['data']
|
||||
new_state = update_data['current_state']
|
||||
if self.fsm_state != new_state:
|
||||
self.handle_fsm_step(**update_data)
|
||||
self.fsm_state = new_state
|
||||
self.fsm_in_accepted_state = update_data['in_accepted_state']
|
||||
self.fsm_event_log.append(update_data)
|
||||
except KeyError:
|
||||
LOG.error('Invalid fsm_update message received!')
|
||||
|
||||
def handle_fsm_step(self, **kwargs):
|
||||
"""
|
||||
Called in case the TFW FSM has stepped.
|
||||
|
||||
:param kwargs: fsm_update 'data' field
|
||||
"""
|
||||
pass
|
@ -1,7 +1,7 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from tfw.networking.event_handlers import ServerUplinkConnector
|
||||
from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
|
||||
|
||||
|
||||
class MessageSender:
|
||||
|
@ -1,5 +1,2 @@
|
||||
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
|
||||
# All Rights Reserved. See LICENSE file for details.
|
||||
|
||||
from .event_handler_connector import EventHandlerConnector, EventHandlerUplinkConnector, EventHandlerDownlinkConnector
|
||||
from .tfw_server import TFWServer
|
||||
|
@ -4,7 +4,8 @@
|
||||
import zmq
|
||||
from zmq.eventloop.zmqstream import ZMQStream
|
||||
|
||||
from tfw.networking import ZMQConnectorBase, serialize_tfw_msg, with_deserialize_tfw_msg
|
||||
from tfw.networking.zmq_connector_base import ZMQConnectorBase
|
||||
from tfw.networking.serialization import serialize_tfw_msg, with_deserialize_tfw_msg
|
||||
from tfw.config import TFWENV
|
||||
from tfw.config.logs import logging
|
||||
|
||||
|
@ -6,23 +6,25 @@ from contextlib import suppress
|
||||
|
||||
from tornado.web import Application
|
||||
|
||||
from tfw.networking.event_handlers import ServerUplinkConnector
|
||||
from tfw.networking.server import EventHandlerConnector
|
||||
from tfw.networking import MessageSender
|
||||
from tfw.networking.server.zmq_websocket_proxy import ZMQWebSocketProxy
|
||||
from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
|
||||
from tfw.networking.server.event_handler_connector import EventHandlerConnector
|
||||
from tfw.networking.message_sender import MessageSender
|
||||
from tfw.networking.fsm_aware import FSMAware
|
||||
from tfw.crypto import KeyManager, verify_message, sign_message
|
||||
from tfw.config.logs import logging
|
||||
from .zmq_websocket_proxy import ZMQWebSocketProxy
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TFWServer:
|
||||
class TFWServer(FSMAware):
|
||||
"""
|
||||
This class handles the proxying of messages between the frontend and event handers.
|
||||
It proxies messages from the "/ws" route to all event handlers subscribed to a ZMQ
|
||||
SUB socket.
|
||||
"""
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._event_handler_connector = EventHandlerConnector()
|
||||
self._uplink_connector = ServerUplinkConnector()
|
||||
self._auth_key = KeyManager().auth_key
|
||||
@ -30,9 +32,16 @@ class TFWServer:
|
||||
self.application = Application([(
|
||||
r'/ws', ZMQWebSocketProxy, {
|
||||
'event_handler_connector': self._event_handler_connector,
|
||||
'message_handlers': [self.handle_trigger, self.handle_recover],
|
||||
'frontend_message_handlers': [self.save_frontend_messages]
|
||||
})])
|
||||
'proxy_filters_and_callbacks': {
|
||||
'message_handlers': [
|
||||
self.handle_trigger,
|
||||
self.handle_recover,
|
||||
self.handle_fsm_update
|
||||
],
|
||||
'frontend_message_handlers': [self.save_frontend_messages]
|
||||
}
|
||||
}
|
||||
)])
|
||||
|
||||
self._frontend_messages = FrontendMessageStorage()
|
||||
|
||||
@ -55,6 +64,9 @@ class TFWServer:
|
||||
self._frontend_messages.replay_messages(self._uplink_connector)
|
||||
self._frontend_messages.clear()
|
||||
|
||||
def handle_fsm_update(self, message):
|
||||
self.update_fsm_data(message)
|
||||
|
||||
def save_frontend_messages(self, message):
|
||||
self._frontend_messages.save_message(message)
|
||||
|
||||
|
@ -5,7 +5,7 @@ import json
|
||||
|
||||
from tornado.websocket import WebSocketHandler
|
||||
|
||||
from tfw.mixins import CallbackMixin
|
||||
from tfw.mixins.callback_mixin import CallbackMixin
|
||||
from tfw.config.logs import logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
@ -14,14 +14,11 @@ LOG = logging.getLogger(__name__)
|
||||
class ZMQWebSocketProxy(WebSocketHandler):
|
||||
# pylint: disable=abstract-method
|
||||
instances = set()
|
||||
sequence_number = 0
|
||||
|
||||
def initialize(self, **kwargs): # pylint: disable=arguments-differ
|
||||
self._event_handler_connector = kwargs['event_handler_connector']
|
||||
|
||||
self._message_handlers = kwargs.get('message_handlers', [])
|
||||
self._frontend_message_handlers = kwargs.get('frontend_message_handlers', [])
|
||||
self._eventhandler_message_handlers = kwargs.get('eventhandler_message_handlers', [])
|
||||
self._proxy_filters = kwargs.get('proxy_filters', [])
|
||||
self._proxy_filters_and_callbacks = kwargs.get('proxy_filters_and_callbacks', {})
|
||||
|
||||
self.proxy_eventhandler_to_websocket = TFWProxy(
|
||||
self.send_eventhandler_message,
|
||||
@ -35,14 +32,19 @@ class ZMQWebSocketProxy(WebSocketHandler):
|
||||
self.subscribe_proxy_callbacks()
|
||||
|
||||
def subscribe_proxy_callbacks(self):
|
||||
eventhandler_message_handlers = self._proxy_filters_and_callbacks.get('eventhandler_message_handlers', [])
|
||||
frontend_message_handlers = self._proxy_filters_and_callbacks.get('frontend_message_handlers', [])
|
||||
message_handlers = self._proxy_filters_and_callbacks.get('message_handlers', [])
|
||||
proxy_filters = self._proxy_filters_and_callbacks.get('proxy_filters', [])
|
||||
|
||||
self.proxy_websocket_to_eventhandler.subscribe_proxy_callbacks_and_filters(
|
||||
self._eventhandler_message_handlers + self._message_handlers,
|
||||
self._proxy_filters
|
||||
eventhandler_message_handlers + message_handlers,
|
||||
proxy_filters
|
||||
)
|
||||
|
||||
self.proxy_eventhandler_to_websocket.subscribe_proxy_callbacks_and_filters(
|
||||
self._frontend_message_handlers + self._message_handlers,
|
||||
self._proxy_filters
|
||||
frontend_message_handlers + message_handlers,
|
||||
proxy_filters
|
||||
)
|
||||
|
||||
def prepare(self):
|
||||
@ -59,14 +61,21 @@ class ZMQWebSocketProxy(WebSocketHandler):
|
||||
"""
|
||||
Invoked on ZMQ messages from event handlers.
|
||||
"""
|
||||
self.sequence_message(message)
|
||||
LOG.debug('Received on pull socket: %s', message)
|
||||
self.proxy_eventhandler_to_websocket(message)
|
||||
|
||||
@classmethod
|
||||
def sequence_message(cls, message):
|
||||
cls.sequence_number += 1
|
||||
message['seq'] = cls.sequence_number
|
||||
|
||||
def on_message(self, message):
|
||||
"""
|
||||
Invoked on WS messages from frontend.
|
||||
"""
|
||||
message = json.loads(message)
|
||||
self.sequence_message(message)
|
||||
LOG.debug('Received on WebSocket: %s', message)
|
||||
self.proxy_websocket_to_eventhandler(message)
|
||||
|
||||
@ -105,14 +114,9 @@ class TFWProxy:
|
||||
raise ValueError('Invalid TFW message format!')
|
||||
|
||||
def __call__(self, message):
|
||||
try:
|
||||
self.proxy_filters._execute_callbacks(message)
|
||||
except ValueError:
|
||||
LOG.exception('Invalid TFW message received!')
|
||||
if not self.filter_and_execute_callbacks(message):
|
||||
return
|
||||
|
||||
self.proxy_callbacks._execute_callbacks(message)
|
||||
|
||||
if message['key'] not in self.keyhandlers:
|
||||
self.to_destination(message)
|
||||
else:
|
||||
@ -122,13 +126,26 @@ class TFWProxy:
|
||||
except KeyError:
|
||||
LOG.error('Invalid "%s" message format! Ignoring.', handler.__name__)
|
||||
|
||||
def filter_and_execute_callbacks(self, message):
|
||||
try:
|
||||
self.proxy_filters._execute_callbacks(message)
|
||||
self.proxy_callbacks._execute_callbacks(message)
|
||||
return True
|
||||
except ValueError:
|
||||
LOG.exception('Invalid TFW message received!')
|
||||
return False
|
||||
|
||||
def mirror(self, message):
|
||||
message = message['data']
|
||||
if not self.filter_and_execute_callbacks(message):
|
||||
return
|
||||
LOG.debug('Mirroring message: %s', message)
|
||||
self.to_source(message)
|
||||
|
||||
def broadcast(self, message):
|
||||
message = message['data']
|
||||
if not self.filter_and_execute_callbacks(message):
|
||||
return
|
||||
LOG.debug('Broadcasting message: %s', message)
|
||||
self.to_source(message)
|
||||
self.to_destination(message)
|
||||
|
@ -1,8 +1,9 @@
|
||||
tornado==5.0
|
||||
pyzmq==17.0.0
|
||||
transitions==0.6.4
|
||||
tornado==5.1
|
||||
pyzmq==17.1.2
|
||||
transitions==0.6.6
|
||||
terminado==0.8.1
|
||||
watchdog==0.8.3
|
||||
PyYAML==3.12
|
||||
PyYAML==3.13
|
||||
Jinja2==2.10
|
||||
cryptography==2.2.2
|
||||
cryptography==2.3.1
|
||||
python-dateutil==2.7.3
|
||||
|
Loading…
Reference in New Issue
Block a user