mrtoth/baseimage-tutorial-framework
1
0
Fork 0
mirror of https://github.com/avatao-content/baseimage-tutorial-framework synced 2025-11-04 02:52:55 +00:00
Code Issues Releases Wiki Activity

Merge branch 'ocicat', the unrealized dream. Ocicat will return...

Browse Source
This commit is contained in:
Kristóf Tóth
2019-05-15 11:19:24 +02:00
parent f43c463692 1d969f0967
commit 07cd1264f5
55 changed files with 715 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/envvars/__init__.py
View File

@@ -4,7 +4,7 @@
from collections import namedtuple
from os import environ
from tfw.decorators import lazy_property
from tfw.decorators.lazy_property import lazy_property
class LazyEnvironment:

5
lib/tfw/__init__.py
View File

@@ -1,7 +1,2 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from .event_handler_base import EventHandlerBase, FSMAwareEventHandler, BroadcastingEventHandler
from .fsm_base import FSMBase
from .linear_fsm import LinearFSM
from .yaml_fsm import YamlFSM

2
lib/tfw/components/__init__.py
View File

@@ -12,3 +12,5 @@ from .fsm_managing_event_handler import FSMManagingEventHandler
from .snapshot_provider import SnapshotProvider
from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler, PipeIOServer
from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler
from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler
from .commands_equal import CommandsEqual

110
lib/tfw/components/commands_equal.py Normal file
View File

@@ -0,0 +1,110 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from shlex import split
from re import search
from tfw.decorators.lazy_property import lazy_property
class CommandsEqual:
# pylint: disable=too-many-arguments
"""
This class is useful for comparing executed commands with
excepted commands (i.e. when triggering a state change when
the correct command is executed).
Note that in most cases you should test the changes
caused by the commands instead of just checking command history
(stuff can be done in countless ways and preparing for every
single case is impossible). This should only be used when
testing the changes would be very difficult, like when
explaining stuff with cli tools and such.
This class implicitly converts to bool, use it like
if CommandsEqual(...): ...
It tries detecting differing command parameter orders with similar
semantics and provides fuzzy logic options.
The rationale behind this is that a few false positives
are better than only accepting a single version of a command
(i.e. using ==).
"""
def __init__(
self, command_1, command_2,
fuzzyness=1, begin_similarly=True,
include_patterns=None, exclude_patterns=None
):
"""
:param command_1: Compared command 1
:param command_2: Compared command 2
:param fuzzyness: float between 0 and 1.
the percentage of arguments required to
match between commands to result in True.
i.e 1 means 100% - all arguments need to be
present in both commands, while 0.75
would mean 75% - in case of 4 arguments
1 could differ between the commands.
:param begin_similarly: bool, the first word of the commands
must match
:param include_patterns: list of regex patterns the commands
must include
:param exclude_patterns: list of regex patterns the commands
must exclude
"""
self.command_1 = split(command_1)
self.command_2 = split(command_2)
self.fuzzyness = fuzzyness
self.begin_similarly = begin_similarly
self.include_patterns = include_patterns
self.exclude_patterns = exclude_patterns
def __bool__(self):
if self.begin_similarly:
if not self.beginnings_are_equal:
return False
if self.include_patterns is not None:
if not self.commands_contain_include_patterns:
return False
if self.exclude_patterns is not None:
if not self.commands_contain_no_exclude_patterns:
return False
return self.similarity >= self.fuzzyness
@lazy_property
def beginnings_are_equal(self):
return self.command_1[0] == self.command_2[0]
@lazy_property
def commands_contain_include_patterns(self):
return all((
self.contains_regex_patterns(self.command_1, self.include_patterns),
self.contains_regex_patterns(self.command_2, self.include_patterns)
))
@lazy_property
def commands_contain_no_exclude_patterns(self):
return all((
not self.contains_regex_patterns(self.command_1, self.exclude_patterns),
not self.contains_regex_patterns(self.command_2, self.exclude_patterns)
))
@staticmethod
def contains_regex_patterns(command, regex_parts):
command = ' '.join(command)
for pattern in regex_parts:
if not search(pattern, command):
return False
return True
@lazy_property
def similarity(self):
parts_1 = set(self.command_1)
parts_2 = set(self.command_2)
difference = parts_1 - parts_2
deviance = len(difference) / len(max(parts_1, parts_2))
return 1 - deviance

6
lib/tfw/components/directory_monitor.py
View File

@@ -5,9 +5,9 @@ from functools import wraps
from watchdog.events import FileSystemEventHandler as FileSystemWatchdogEventHandler
from tfw.networking.event_handlers import ServerUplinkConnector
from tfw.decorators import RateLimiter
from tfw.mixins import ObserverMixin
from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
from tfw.decorators.rate_limiter import RateLimiter
from tfw.mixins.observer_mixin import ObserverMixin
from tfw.config.logs import logging

6
lib/tfw/components/directory_monitoring_event_handler.py
View File

@@ -3,10 +3,10 @@
from os.path import isdir, exists
from tfw import EventHandlerBase
from tfw.event_handler_base import EventHandlerBase
from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin
from tfw.components.directory_monitor import DirectoryMonitor
from tfw.config.logs import logging
from tfw.mixins import MonitorManagerMixin
from .directory_monitor import DirectoryMonitor
LOG = logging.getLogger(__name__)

87
lib/tfw/components/directory_snapshotting_event_handler.py Normal file
View File

@@ -0,0 +1,87 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from os.path import join as joinpath
from os.path import basename
from os import makedirs
from datetime import datetime
from dateutil import parser as dateparser
from tfw.event_handler_base import EventHandlerBase
from tfw.components.snapshot_provider import SnapshotProvider
from tfw.config import TFWENV
from tfw.config.logs import logging
LOG = logging.getLogger(__name__)
class DirectorySnapshottingEventHandler(EventHandlerBase):
def __init__(self, key, directories, exclude_unix_patterns=None):
super().__init__(key)
self.snapshot_providers = {}
self._exclude_unix_patterns = exclude_unix_patterns
self.init_snapshot_providers(directories)
self.command_handlers = {
'take_snapshot': self.handle_take_snapshot,
'restore_snapshot': self.handle_restore_snapshot,
'exclude': self.handle_exclude
}
def init_snapshot_providers(self, directories):
for index, directory in enumerate(directories):
git_dir = self.init_git_dir(index, directory)
self.snapshot_providers[directory] = SnapshotProvider(
directory,
git_dir,
self._exclude_unix_patterns
)
@staticmethod
def init_git_dir(index, directory):
git_dir = joinpath(
TFWENV.SNAPSHOTS_DIR,
f'{basename(directory)}-{index}'
)
makedirs(git_dir, exist_ok=True)
return git_dir
def handle_event(self, message):
try:
data = message['data']
message['data'] = self.command_handlers[data['command']](data)
return message
except KeyError:
LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
def handle_take_snapshot(self, data):
LOG.debug('Taking snapshots of directories %s', self.snapshot_providers.keys())
for provider in self.snapshot_providers.values():
provider.take_snapshot()
return data
def handle_restore_snapshot(self, data):
date = dateparser.parse(
data.get(
'value',
datetime.now().isoformat()
)
)
LOG.debug(
'Restoring snapshots (@ %s) of directories %s',
date,
self.snapshot_providers.keys()
)
for provider in self.snapshot_providers.values():
provider.restore_snapshot(date)
return data
def handle_exclude(self, data):
exclude_unix_patterns = data['value']
if not isinstance(exclude_unix_patterns, list):
raise KeyError
for provider in self.snapshot_providers.values():
provider.exclude = exclude_unix_patterns
return data

46
lib/tfw/components/fsm_managing_event_handler.py
View File

@@ -1,7 +1,7 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from tfw import EventHandlerBase
from tfw.event_handler_base import EventHandlerBase
from tfw.crypto import KeyManager, sign_message, verify_message
from tfw.config.logs import logging
@@ -9,6 +9,20 @@ LOG = logging.getLogger(__name__)
class FSMManagingEventHandler(EventHandlerBase):
"""
EventHandler responsible for managing the state machine of
the framework (TFW FSM).
tfw.networking.TFWServer instances automatically send 'trigger'
commands to the event handler listening on the 'fsm' key,
which should be an instance of this event handler.
This event handler accepts messages that have a
data['command'] key specifying a command to be executed.
An 'fsm_update' message is broadcasted after every successful
command.
"""
def __init__(self, key, fsm_type, require_signature=False):
super().__init__(key)
self.fsm = fsm_type()
@@ -25,7 +39,7 @@ class FSMManagingEventHandler(EventHandlerBase):
try:
message = self.command_handlers[message['data']['command']](message)
if message:
fsm_update_message = self._fsm_updater.generate_fsm_update()
fsm_update_message = self._fsm_updater.fsm_update
sign_message(self.auth_key, message)
sign_message(self.auth_key, fsm_update_message)
self.server_connector.broadcast(fsm_update_message)
@@ -34,6 +48,12 @@ class FSMManagingEventHandler(EventHandlerBase):
LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
def handle_trigger(self, message):
"""
Attempts to step the FSM with the supplied trigger.
:param message: TFW message with a data field containing
the action to try triggering in data['value']
"""
trigger = message['data']['value']
if self._require_signature:
if not verify_message(self.auth_key, message):
@@ -44,6 +64,9 @@ class FSMManagingEventHandler(EventHandlerBase):
return None
def handle_update(self, message):
"""
Does nothing, but triggers an 'fsm_update' message.
"""
# pylint: disable=no-self-use
return message
@@ -52,23 +75,24 @@ class FSMUpdater:
def __init__(self, fsm):
self.fsm = fsm
def generate_fsm_update(self):
@property
def fsm_update(self):
return {
'key': 'fsm_update',
'data': self.get_fsm_state_and_transitions()
'data': self.fsm_update_data
}
def get_fsm_state_and_transitions(self):
state = self.fsm.state
@property
def fsm_update_data(self):
valid_transitions = [
{'trigger': trigger}
for trigger in self.fsm.get_triggers(self.fsm.state)
]
last_trigger = self.fsm.trigger_history[-1] if self.fsm.trigger_history else None
in_accepted_state = state in self.fsm.accepted_states
last_fsm_event = self.fsm.event_log[-1]
last_fsm_event['timestamp'] = last_fsm_event['timestamp'].isoformat()
return {
'current_state': state,
'current_state': self.fsm.state,
'valid_transitions': valid_transitions,
'last_trigger': last_trigger,
'in_accepted_state': in_accepted_state
'in_accepted_state': self.fsm.in_accepted_state,
'last_event': last_fsm_event
}

5
lib/tfw/components/history_monitor.py
View File

@@ -8,8 +8,9 @@ from abc import ABC, abstractmethod
from watchdog.events import PatternMatchingEventHandler
from tfw.mixins import CallbackMixin, ObserverMixin
from tfw.decorators import RateLimiter
from tfw.mixins.callback_mixin import CallbackMixin
from tfw.mixins.observer_mixin import ObserverMixin
from tfw.decorators.rate_limiter import RateLimiter
class CallbackEventHandler(PatternMatchingEventHandler, ABC):

9
lib/tfw/components/ide_event_handler.py
View File

@@ -6,10 +6,10 @@ from glob import glob
from fnmatch import fnmatchcase
from typing import Iterable
from tfw import EventHandlerBase
from tfw.mixins import MonitorManagerMixin
from tfw.event_handler_base import EventHandlerBase
from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin
from tfw.components.directory_monitor import DirectoryMonitor
from tfw.config.logs import logging
from .directory_monitor import DirectoryMonitor
LOG = logging.getLogger(__name__)
@@ -157,7 +157,8 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
"""
Read the currently selected file.
:return dict: message with the contents of the file in data['content']
:return dict: TFW message data containing key 'content'
(contents of the selected file)
"""
try:
data['content'] = self.filemanager.file_contents

7
lib/tfw/components/log_monitor.py
View File

@@ -6,9 +6,10 @@ from os.path import dirname
from watchdog.events import PatternMatchingEventHandler as PatternMatchingWatchdogEventHandler
from tfw.networking.event_handlers import ServerUplinkConnector
from tfw.decorators import RateLimiter
from tfw.mixins import ObserverMixin, SupervisorLogMixin
from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
from tfw.decorators.rate_limiter import RateLimiter
from tfw.mixins.observer_mixin import ObserverMixin
from tfw.mixins.supervisor_mixin import SupervisorLogMixin
class LogMonitor(ObserverMixin):

6
lib/tfw/components/log_monitoring_event_handler.py
View File

@@ -1,10 +1,10 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from tfw import EventHandlerBase
from tfw.mixins import MonitorManagerMixin
from tfw.event_handler_base import EventHandlerBase
from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin
from tfw.components.log_monitor import LogMonitor
from tfw.config.logs import logging
from .log_monitor import LogMonitor
LOG = logging.getLogger(__name__)

2
lib/tfw/components/pipe_io_event_handler.py
View File

@@ -9,7 +9,7 @@ from secrets import token_urlsafe
from threading import Thread
from contextlib import suppress
from tfw import EventHandlerBase
from tfw.event_handler_base import EventHandlerBase
from tfw.config.logs import logging
from .pipe_io_server import PipeIOServer, terminate_process_on_failure

6
lib/tfw/components/process_managing_event_handler.py
View File

@@ -3,10 +3,10 @@
from xmlrpc.client import Fault as SupervisorFault
from tfw import EventHandlerBase
from tfw.mixins import SupervisorMixin, SupervisorLogMixin
from tfw.event_handler_base import EventHandlerBase
from tfw.mixins.supervisor_mixin import SupervisorMixin, SupervisorLogMixin
from tfw.components.directory_monitor import with_monitor_paused
from tfw.config.logs import logging
from .directory_monitor import with_monitor_paused
LOG = logging.getLogger(__name__)

71
lib/tfw/components/snapshot_provider.py
View File

@@ -2,15 +2,17 @@
# All Rights Reserved. See LICENSE file for details.
import re
from subprocess import run, CalledProcessError
from subprocess import run, CalledProcessError, PIPE
from getpass import getuser
from os.path import isdir
from datetime import datetime
from os.path import join as joinpath
from uuid import uuid4
from dateutil import parser as dateparser
class SnapshotProvider:
def __init__(self, directory, git_dir):
def __init__(self, directory, git_dir, exclude_unix_patterns=None):
self._classname = self.__class__.__name__
author = f'{getuser()} via TFW {self._classname}'
self.gitenv = {
@@ -25,6 +27,8 @@ class SnapshotProvider:
self._init_repo()
self.__last_valid_branch = self._branch
if exclude_unix_patterns:
self.exclude = exclude_unix_patterns
def _init_repo(self):
self._check_environment()
@@ -66,10 +70,14 @@ class SnapshotProvider:
'git', 'add',
'-A'
))
self._run((
'git', 'commit',
'-m', 'Snapshot'
))
try:
self._get_stdout((
'git', 'commit',
'-m', 'Snapshot'
))
except CalledProcessError as err:
if b'nothing to commit, working tree clean' not in err.output:
raise
def _check_head_not_detached(self):
if self._head_detached:
@@ -87,7 +95,8 @@ class SnapshotProvider:
))
def _get_stdout(self, *args, **kwargs):
kwargs['capture_output'] = True
kwargs['stdout'] = PIPE
kwargs['stderr'] = PIPE
stdout_bytes = self._run(*args, **kwargs).stdout
return stdout_bytes.decode().rstrip('\n')
@@ -98,13 +107,31 @@ class SnapshotProvider:
kwargs['env'] = self.gitenv
return run(*args, **kwargs)
@property
def exclude(self):
with open(self._exclude_path, 'r') as ofile:
return ofile.read()
@exclude.setter
def exclude(self, exclude_patterns):
with open(self._exclude_path, 'w') as ifile:
ifile.write('\n'.join(exclude_patterns))
@property
def _exclude_path(self):
return joinpath(
self.gitenv['GIT_DIR'],
'info',
'exclude'
)
def take_snapshot(self):
if self._head_detached:
self._checkout_new_branch_from_head()
self._snapshot()
def _checkout_new_branch_from_head(self):
branch_name = uuid4()
branch_name = str(uuid4())
self._run((
'git', 'branch',
branch_name
@@ -119,16 +146,30 @@ class SnapshotProvider:
def restore_snapshot(self, date):
commit = self._get_commit_from_timestamp(date)
branch = self._last_valid_branch
if commit == self._latest_commit_on_branch(branch):
commit = branch
self._checkout(commit)
def _get_commit_from_timestamp(self, date):
return self._get_stdout((
commit = self._get_stdout((
'git', 'rev-list',
'--date=iso',
'-n', '1',
f'--before="{date.isoformat()}"',
self._last_valid_branch
))
if not commit:
commit = self._get_oldest_parent_of_head()
return commit
def _get_oldest_parent_of_head(self):
return self._get_stdout((
'git',
'rev-list',
'--max-parents=0',
'HEAD'
))
@property
def _last_valid_branch(self):
@@ -136,6 +177,14 @@ class SnapshotProvider:
self.__last_valid_branch = self._branch
return self.__last_valid_branch
def _latest_commit_on_branch(self, branch):
return self._get_stdout((
'git', 'log',
'-n', '1',
'--pretty=format:%H',
branch
))
@property
def all_timelines(self):
return self._branches
@@ -169,7 +218,7 @@ class SnapshotProvider:
commit_hash, timestamp = line.split('@')
commits.append({
'hash': commit_hash,
'timestamp': datetime.fromisoformat(timestamp)
'timestamp': dateparser.parse(timestamp)
})
return commits

5
lib/tfw/components/terminal_event_handler.py
View File

@@ -1,11 +1,11 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from tfw import EventHandlerBase
from tfw.event_handler_base import EventHandlerBase
from tfw.components.terminado_mini_server import TerminadoMiniServer
from tfw.config import TFWENV
from tfw.config.logs import logging
from tao.config import TAOENV
from .terminado_mini_server import TerminadoMiniServer
LOG = logging.getLogger(__name__)
@@ -26,7 +26,6 @@ class TerminalEventHandler(EventHandlerBase):
:param monitor: tfw.components.HistoryMonitor instance to read command history from
"""
super().__init__(key)
self.working_directory = TFWENV.TERMINADO_DIR
self._historymonitor = monitor
bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash']

4
lib/tfw/crypto.py
View File

@@ -14,8 +14,8 @@ from cryptography.hazmat.primitives.hashes import SHA256
from cryptography.hazmat.primitives.hmac import HMAC as _HMAC
from cryptography.exceptions import InvalidSignature
from tfw.networking import message_bytes
from tfw.decorators import lazy_property
from tfw.networking.serialization import message_bytes
from tfw.decorators.lazy_property import lazy_property
from tfw.config import TFWENV

3
lib/tfw/decorators/__init__.py
View File

@@ -1,5 +1,2 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from .rate_limiter import RateLimiter
from .lazy_property import lazy_property

11
lib/tfw/decorators/lazy_property.py
View File

@@ -1,7 +1,7 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from functools import update_wrapper
from functools import update_wrapper, wraps
class lazy_property:
@@ -19,3 +19,12 @@ class lazy_property:
value = self.func(instance)
setattr(instance, self.func.__name__, value)
return value
def lazy_factory(fun):
class wrapper:
@wraps(fun)
@lazy_property
def instance(self): # pylint: disable=no-self-use
return fun()
return wrapper()

1
lib/tfw/decorators/rate_limiter.py
View File

@@ -87,6 +87,7 @@ class AsyncRateLimiter(RateLimiter):
return self._ioloop_factory()
def action(self, seconds_to_next_call):
# pylint: disable=method-hidden
if self._last_callback:
self.ioloop.remove_timeout(self._last_callback)

6
lib/tfw/event_handler_base/__init__.py Normal file
View File

@@ -0,0 +1,6 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from .event_handler_base import EventHandlerBase
from .boradcasting_event_handler import BroadcastingEventHandler
from .fsm_aware_event_handler import FSMAwareEventHandler

30
lib/tfw/event_handler_base/boradcasting_event_handler.py Normal file
View File

@@ -0,0 +1,30 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from abc import ABC
from tfw.event_handler_base.event_handler_base import EventHandlerBase
from tfw.crypto import message_checksum
class BroadcastingEventHandler(EventHandlerBase, ABC):
# pylint: disable=abstract-method
"""
Abstract base class for EventHandlers which broadcast responses
and intelligently ignore their own broadcasted messages they receive.
"""
def __init__(self, key):
super().__init__(key)
self.own_message_hashes = []
def event_handler_callback(self, message):
message_hash = message_checksum(message)
if message_hash in self.own_message_hashes:
self.own_message_hashes.remove(message_hash)
return
response = self.dispatch_handling(message)
if response:
self.own_message_hashes.append(message_checksum(response))
self.server_connector.broadcast(response)

64
lib/tfw/event_handler_base.py → lib/tfw/event_handler_base/event_handler_base.py
View File

@@ -5,8 +5,7 @@ from abc import ABC, abstractmethod
from inspect import currentframe
from typing import Iterable
from tfw.networking.event_handlers import ServerConnector
from tfw.crypto import message_checksum, KeyManager, verify_message
from tfw.networking.event_handlers.server_connector import ServerConnector
from tfw.config.logs import logging
LOG = logging.getLogger(__name__)
@@ -125,64 +124,3 @@ class EventHandlerBase(ABC):
instance for instance in locals_values
if isinstance(instance, cls)
}
class FSMAwareEventHandler(EventHandlerBase, ABC):
# pylint: disable=abstract-method
"""
Abstract base class for EventHandlers which automatically
keep track of the state of the TFW FSM.
"""
def __init__(self, key):
super().__init__(key)
self.subscribe('fsm_update')
self.fsm_state = None
self.in_accepted_state = False
self._auth_key = KeyManager().auth_key
def dispatch_handling(self, message):
if message['key'] == 'fsm_update':
if verify_message(self._auth_key, message):
self._handle_fsm_update(message)
return None
return super().dispatch_handling(message)
def _handle_fsm_update(self, message):
try:
new_state = message['data']['current_state']
trigger = message['data']['last_trigger']
if self.fsm_state != new_state:
self.handle_fsm_step(self.fsm_state, new_state, trigger)
self.fsm_state = new_state
self.in_accepted_state = message['data']['in_accepted_state']
except KeyError:
LOG.error('Invalid fsm_update message received!')
def handle_fsm_step(self, from_state, to_state, trigger):
"""
Called in case the TFW FSM has stepped.
"""
pass
class BroadcastingEventHandler(EventHandlerBase, ABC):
# pylint: disable=abstract-method
"""
Abstract base class for EventHandlers which broadcast responses
and intelligently ignore their own broadcasted messages they receive.
"""
def __init__(self, key):
super().__init__(key)
self.own_message_hashes = []
def event_handler_callback(self, message):
message_hash = message_checksum(message)
if message_hash in self.own_message_hashes:
self.own_message_hashes.remove(message_hash)
return
response = self.dispatch_handling(message)
if response:
self.own_message_hashes.append(message_checksum(response))
self.server_connector.broadcast(response)

24
lib/tfw/event_handler_base/fsm_aware_event_handler.py Normal file
View File

@@ -0,0 +1,24 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from abc import ABC
from tfw.event_handler_base.event_handler_base import EventHandlerBase
from tfw.networking.fsm_aware import FSMAware
class FSMAwareEventHandler(EventHandlerBase, FSMAware, ABC):
# pylint: disable=abstract-method
"""
Abstract base class for EventHandlers which automatically
keep track of the state of the TFW FSM.
"""
def __init__(self, key):
EventHandlerBase.__init__(self, key)
FSMAware.__init__(self)
self.subscribe('fsm_update')
def dispatch_handling(self, message):
if self.update_fsm_data(message):
return None
return super().dispatch_handling(message)

6
lib/tfw/fsm/__init__.py Normal file
View File

@@ -0,0 +1,6 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from .fsm_base import FSMBase
from .linear_fsm import LinearFSM
from .yaml_fsm import YamlFSM

25
lib/tfw/fsm_base.py → lib/tfw/fsm/fsm_base.py
View File

@@ -2,10 +2,11 @@
# All Rights Reserved. See LICENSE file for details.
from collections import defaultdict
from datetime import datetime
from transitions import Machine, MachineError
from tfw.mixins import CallbackMixin
from tfw.mixins.callback_mixin import CallbackMixin
from tfw.config.logs import logging
LOG = logging.getLogger(__name__)
@@ -22,9 +23,14 @@ class FSMBase(Machine, CallbackMixin):
states, transitions = [], []
def __init__(self, initial=None, accepted_states=None):
"""
:param initial: which state to begin with, defaults to the last one
:param accepted_states: list of states in which the challenge should be
considered successfully completed
"""
self.accepted_states = accepted_states or [self.states[-1].name]
self.trigger_predicates = defaultdict(list)
self.trigger_history = []
self.event_log = []
Machine.__init__(
self,
@@ -60,9 +66,22 @@ class FSMBase(Machine, CallbackMixin):
if all(predicate_results):
try:
from_state = self.state
self.trigger(trigger)
self.trigger_history.append(trigger)
self.update_event_log(from_state, trigger)
return True
except (AttributeError, MachineError):
LOG.debug('FSM failed to execute nonexistent trigger: "%s"', trigger)
return False
def update_event_log(self, from_state, trigger):
self.event_log.append({
'from_state': from_state,
'to_state': self.state,
'trigger': trigger,
'timestamp': datetime.utcnow()
})
@property
def in_accepted_state(self):
return self.state in self.accepted_states

8
lib/tfw/linear_fsm.py → lib/tfw/fsm/linear_fsm.py
View File

@@ -3,7 +3,7 @@
from transitions import State
from .fsm_base import FSMBase
from tfw.fsm.fsm_base import FSMBase
class LinearFSM(FSMBase):
@@ -12,9 +12,13 @@ class LinearFSM(FSMBase):
This is a state machine for challenges with linear progression, consisting of
a number of steps specified in the constructor. It automatically sets up 2
actions (triggers) between states as such:
(0) -- step_1 --> (1) -- step_2 --> (2) -- step_3 --> (3) ... and so on
(0) -- step_1 --> (1) -- step_2 --> (2) -- step_3 --> (3) ...
(0) -- step_next --> (1) -- step_next --> (2) -- step_next --> (3) ...
"""
def __init__(self, number_of_steps):
"""
:param number_of_steps: how many states this FSM should have
"""
self.states = [State(name=str(index)) for index in range(number_of_steps)]
self.transitions = []
for state in self.states[:-1]:

17
lib/tfw/yaml_fsm.py → lib/tfw/fsm/yaml_fsm.py
View File

@@ -1,3 +1,6 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from subprocess import Popen, run
from functools import partial, singledispatch
from contextlib import suppress
@@ -6,11 +9,21 @@ import yaml
import jinja2
from transitions import State
from tfw import FSMBase
from tfw.fsm.fsm_base import FSMBase
class YamlFSM(FSMBase):
"""
This is a state machine capable of building itself from a YAML config file.
"""
def __init__(self, config_file, jinja2_variables=None):
"""
:param config_file: path of the YAML file
:param jinja2_variables: dict containing jinja2 variables
or str with filename of YAML file to
parse and use as dict.
jinja2 support is disabled if this is None
"""
self.config = ConfigParser(config_file, jinja2_variables).config
self.setup_states()
super().__init__() # FSMBase.__init__() requires states
@@ -45,7 +58,7 @@ class YamlFSM(FSMBase):
partial(
command_statuscode_is_zero,
predicate
)
)
)
with suppress(KeyError):

5
lib/tfw/mixins/__init__.py
View File

@@ -1,7 +1,2 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from .supervisor_mixin import SupervisorMixin, SupervisorLogMixin
from .callback_mixin import CallbackMixin
from .observer_mixin import ObserverMixin
from .monitor_manager_mixin import MonitorManagerMixin

2
lib/tfw/mixins/callback_mixin.py
View File

@@ -3,7 +3,7 @@
from functools import partial
from tfw.decorators import lazy_property
from tfw.decorators.lazy_property import lazy_property
class CallbackMixin:

2
lib/tfw/mixins/observer_mixin.py
View File

@@ -3,7 +3,7 @@
from watchdog.observers import Observer
from tfw.decorators import lazy_property
from tfw.decorators.lazy_property import lazy_property
class ObserverMixin:

2
lib/tfw/mixins/supervisor_mixin.py
View File

@@ -6,7 +6,7 @@ from xmlrpc.client import Fault as SupervisorFault
from contextlib import suppress
from os import remove
from tfw.decorators import lazy_property
from tfw.decorators.lazy_property import lazy_property
from tfw.config import TFWENV

3
lib/tfw/networking/__init__.py
View File

@@ -1,9 +1,6 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from .serialization import serialize_tfw_msg, deserialize_tfw_msg
from .serialization import with_deserialize_tfw_msg, message_bytes
from .zmq_connector_base import ZMQConnectorBase
from .message_sender import MessageSender
from .event_handlers.server_connector import ServerUplinkConnector as TFWServerConnector
from .server.tfw_server import TFWServer

2
lib/tfw/networking/event_handlers/__init__.py
View File

@@ -1,4 +1,2 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from .server_connector import ServerConnector, ServerUplinkConnector, ServerDownlinkConnector

4
lib/tfw/networking/event_handlers/server_connector.py
View File

@@ -6,8 +6,8 @@ from functools import partial
import zmq
from zmq.eventloop.zmqstream import ZMQStream
from tfw.networking import serialize_tfw_msg, with_deserialize_tfw_msg
from tfw.networking import ZMQConnectorBase
from tfw.networking.zmq_connector_base import ZMQConnectorBase
from tfw.networking.serialization import serialize_tfw_msg, with_deserialize_tfw_msg
from tfw.config import TFWENV
from tfw.config.logs import logging

46
lib/tfw/networking/fsm_aware.py Normal file
View File

@@ -0,0 +1,46 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from tfw.crypto import KeyManager, verify_message
from tfw.config.logs import logging
LOG = logging.getLogger(__name__)
class FSMAware:
"""
Base class for stuff that has to be aware of the framework FSM.
This is done by processing 'fsm_update' messages.
"""
def __init__(self):
self.fsm_state = None
self.fsm_in_accepted_state = False
self.fsm_event_log = []
self._auth_key = KeyManager().auth_key
def update_fsm_data(self, message):
if message['key'] == 'fsm_update' and verify_message(self._auth_key, message):
self._handle_fsm_update(message)
return True
return False
def _handle_fsm_update(self, message):
try:
update_data = message['data']
new_state = update_data['current_state']
if self.fsm_state != new_state:
self.handle_fsm_step(**update_data)
self.fsm_state = new_state
self.fsm_in_accepted_state = update_data['in_accepted_state']
self.fsm_event_log.append(update_data)
except KeyError:
LOG.error('Invalid fsm_update message received!')
def handle_fsm_step(self, **kwargs):
"""
Called in case the TFW FSM has stepped.
:param kwargs: fsm_update 'data' field
"""
pass

2
lib/tfw/networking/message_sender.py
View File

@@ -1,7 +1,7 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from tfw.networking.event_handlers import ServerUplinkConnector
from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
class MessageSender:

3
lib/tfw/networking/server/__init__.py
View File

@@ -1,5 +1,2 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from .event_handler_connector import EventHandlerConnector, EventHandlerUplinkConnector, EventHandlerDownlinkConnector
from .tfw_server import TFWServer

3
lib/tfw/networking/server/event_handler_connector.py
View File

@@ -4,7 +4,8 @@
import zmq
from zmq.eventloop.zmqstream import ZMQStream
from tfw.networking import ZMQConnectorBase, serialize_tfw_msg, with_deserialize_tfw_msg
from tfw.networking.zmq_connector_base import ZMQConnectorBase
from tfw.networking.serialization import serialize_tfw_msg, with_deserialize_tfw_msg
from tfw.config import TFWENV
from tfw.config.logs import logging

28
lib/tfw/networking/server/tfw_server.py
View File

@@ -6,23 +6,25 @@ from contextlib import suppress
from tornado.web import Application
from tfw.networking.event_handlers import ServerUplinkConnector
from tfw.networking.server import EventHandlerConnector
from tfw.networking import MessageSender
from tfw.networking.server.zmq_websocket_proxy import ZMQWebSocketProxy
from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
from tfw.networking.server.event_handler_connector import EventHandlerConnector
from tfw.networking.message_sender import MessageSender
from tfw.networking.fsm_aware import FSMAware
from tfw.crypto import KeyManager, verify_message, sign_message
from tfw.config.logs import logging
from .zmq_websocket_proxy import ZMQWebSocketProxy
LOG = logging.getLogger(__name__)
class TFWServer:
class TFWServer(FSMAware):
"""
This class handles the proxying of messages between the frontend and event handers.
It proxies messages from the "/ws" route to all event handlers subscribed to a ZMQ
SUB socket.
"""
def __init__(self):
super().__init__()
self._event_handler_connector = EventHandlerConnector()
self._uplink_connector = ServerUplinkConnector()
self._auth_key = KeyManager().auth_key
@@ -30,9 +32,16 @@ class TFWServer:
self.application = Application([(
r'/ws', ZMQWebSocketProxy, {
'event_handler_connector': self._event_handler_connector,
'message_handlers': [self.handle_trigger, self.handle_recover],
'frontend_message_handlers': [self.save_frontend_messages]
})])
'proxy_filters_and_callbacks': {
'message_handlers': [
self.handle_trigger,
self.handle_recover,
self.handle_fsm_update
],
'frontend_message_handlers': [self.save_frontend_messages]
}
}
)])
self._frontend_messages = FrontendMessageStorage()
@@ -55,6 +64,9 @@ class TFWServer:
self._frontend_messages.replay_messages(self._uplink_connector)
self._frontend_messages.clear()
def handle_fsm_update(self, message):
self.update_fsm_data(message)
def save_frontend_messages(self, message):
self._frontend_messages.save_message(message)

49
lib/tfw/networking/server/zmq_websocket_proxy.py
View File

@@ -5,7 +5,7 @@ import json
from tornado.websocket import WebSocketHandler
from tfw.mixins import CallbackMixin
from tfw.mixins.callback_mixin import CallbackMixin
from tfw.config.logs import logging
LOG = logging.getLogger(__name__)
@@ -14,14 +14,11 @@ LOG = logging.getLogger(__name__)
class ZMQWebSocketProxy(WebSocketHandler):
# pylint: disable=abstract-method
instances = set()
sequence_number = 0
def initialize(self, **kwargs): # pylint: disable=arguments-differ
self._event_handler_connector = kwargs['event_handler_connector']
self._message_handlers = kwargs.get('message_handlers', [])
self._frontend_message_handlers = kwargs.get('frontend_message_handlers', [])
self._eventhandler_message_handlers = kwargs.get('eventhandler_message_handlers', [])
self._proxy_filters = kwargs.get('proxy_filters', [])
self._proxy_filters_and_callbacks = kwargs.get('proxy_filters_and_callbacks', {})
self.proxy_eventhandler_to_websocket = TFWProxy(
self.send_eventhandler_message,
@@ -35,14 +32,19 @@ class ZMQWebSocketProxy(WebSocketHandler):
self.subscribe_proxy_callbacks()
def subscribe_proxy_callbacks(self):
eventhandler_message_handlers = self._proxy_filters_and_callbacks.get('eventhandler_message_handlers', [])
frontend_message_handlers = self._proxy_filters_and_callbacks.get('frontend_message_handlers', [])
message_handlers = self._proxy_filters_and_callbacks.get('message_handlers', [])
proxy_filters = self._proxy_filters_and_callbacks.get('proxy_filters', [])
self.proxy_websocket_to_eventhandler.subscribe_proxy_callbacks_and_filters(
self._eventhandler_message_handlers + self._message_handlers,
self._proxy_filters
eventhandler_message_handlers + message_handlers,
proxy_filters
)
self.proxy_eventhandler_to_websocket.subscribe_proxy_callbacks_and_filters(
self._frontend_message_handlers + self._message_handlers,
self._proxy_filters
frontend_message_handlers + message_handlers,
proxy_filters
)
def prepare(self):
@@ -59,14 +61,21 @@ class ZMQWebSocketProxy(WebSocketHandler):
"""
Invoked on ZMQ messages from event handlers.
"""
self.sequence_message(message)
LOG.debug('Received on pull socket: %s', message)
self.proxy_eventhandler_to_websocket(message)
@classmethod
def sequence_message(cls, message):
cls.sequence_number += 1
message['seq'] = cls.sequence_number
def on_message(self, message):
"""
Invoked on WS messages from frontend.
"""
message = json.loads(message)
self.sequence_message(message)
LOG.debug('Received on WebSocket: %s', message)
self.proxy_websocket_to_eventhandler(message)
@@ -105,14 +114,9 @@ class TFWProxy:
raise ValueError('Invalid TFW message format!')
def __call__(self, message):
try:
self.proxy_filters._execute_callbacks(message)
except ValueError:
LOG.exception('Invalid TFW message received!')
if not self.filter_and_execute_callbacks(message):
return
self.proxy_callbacks._execute_callbacks(message)
if message['key'] not in self.keyhandlers:
self.to_destination(message)
else:
@@ -122,13 +126,26 @@ class TFWProxy:
except KeyError:
LOG.error('Invalid "%s" message format! Ignoring.', handler.__name__)
def filter_and_execute_callbacks(self, message):
try:
self.proxy_filters._execute_callbacks(message)
self.proxy_callbacks._execute_callbacks(message)
return True
except ValueError:
LOG.exception('Invalid TFW message received!')
return False
def mirror(self, message):
message = message['data']
if not self.filter_and_execute_callbacks(message):
return
LOG.debug('Mirroring message: %s', message)
self.to_source(message)
def broadcast(self, message):
message = message['data']
if not self.filter_and_execute_callbacks(message):
return
LOG.debug('Broadcasting message: %s', message)
self.to_source(message)
self.to_destination(message)