mrtoth/baseimage-tutorial-framework
1
0
Fork 0
mirror of https://github.com/avatao-content/baseimage-tutorial-framework synced 2025-06-28 17:45:11 +00:00
Code Issues Releases Wiki Activity

Merge branch 'ocicat', the unrealized dream. Ocicat will return...

Browse Source
This commit is contained in:
Kristóf Tóth
2019-05-15 11:19:24 +02:00
parent f43c463692 1d969f0967
commit 07cd1264f5
55 changed files with 715 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/tfw/components/__init__.py
View File

@ -12,3 +12,5 @@ from .fsm_managing_event_handler import FSMManagingEventHandler
from .snapshot_provider import SnapshotProvider
from .pipe_io_event_handler import PipeIOEventHandlerBase, PipeIOEventHandler, PipeIOServer
from .pipe_io_event_handler import TransformerPipeIOEventHandler, CommandEventHandler
from .directory_snapshotting_event_handler import DirectorySnapshottingEventHandler
from .commands_equal import CommandsEqual

110
lib/tfw/components/commands_equal.py Normal file
View File

@ -0,0 +1,110 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from shlex import split
from re import search
from tfw.decorators.lazy_property import lazy_property
class CommandsEqual:
# pylint: disable=too-many-arguments
"""
This class is useful for comparing executed commands with
excepted commands (i.e. when triggering a state change when
the correct command is executed).
Note that in most cases you should test the changes
caused by the commands instead of just checking command history
(stuff can be done in countless ways and preparing for every
single case is impossible). This should only be used when
testing the changes would be very difficult, like when
explaining stuff with cli tools and such.
This class implicitly converts to bool, use it like
if CommandsEqual(...): ...
It tries detecting differing command parameter orders with similar
semantics and provides fuzzy logic options.
The rationale behind this is that a few false positives
are better than only accepting a single version of a command
(i.e. using ==).
"""
def __init__(
self, command_1, command_2,
fuzzyness=1, begin_similarly=True,
include_patterns=None, exclude_patterns=None
):
"""
:param command_1: Compared command 1
:param command_2: Compared command 2
:param fuzzyness: float between 0 and 1.
the percentage of arguments required to
match between commands to result in True.
i.e 1 means 100% - all arguments need to be
present in both commands, while 0.75
would mean 75% - in case of 4 arguments
1 could differ between the commands.
:param begin_similarly: bool, the first word of the commands
must match
:param include_patterns: list of regex patterns the commands
must include
:param exclude_patterns: list of regex patterns the commands
must exclude
"""
self.command_1 = split(command_1)
self.command_2 = split(command_2)
self.fuzzyness = fuzzyness
self.begin_similarly = begin_similarly
self.include_patterns = include_patterns
self.exclude_patterns = exclude_patterns
def __bool__(self):
if self.begin_similarly:
if not self.beginnings_are_equal:
return False
if self.include_patterns is not None:
if not self.commands_contain_include_patterns:
return False
if self.exclude_patterns is not None:
if not self.commands_contain_no_exclude_patterns:
return False
return self.similarity >= self.fuzzyness
@lazy_property
def beginnings_are_equal(self):
return self.command_1[0] == self.command_2[0]
@lazy_property
def commands_contain_include_patterns(self):
return all((
self.contains_regex_patterns(self.command_1, self.include_patterns),
self.contains_regex_patterns(self.command_2, self.include_patterns)
))
@lazy_property
def commands_contain_no_exclude_patterns(self):
return all((
not self.contains_regex_patterns(self.command_1, self.exclude_patterns),
not self.contains_regex_patterns(self.command_2, self.exclude_patterns)
))
@staticmethod
def contains_regex_patterns(command, regex_parts):
command = ' '.join(command)
for pattern in regex_parts:
if not search(pattern, command):
return False
return True
@lazy_property
def similarity(self):
parts_1 = set(self.command_1)
parts_2 = set(self.command_2)
difference = parts_1 - parts_2
deviance = len(difference) / len(max(parts_1, parts_2))
return 1 - deviance

6
lib/tfw/components/directory_monitor.py
View File

@ -5,9 +5,9 @@ from functools import wraps
from watchdog.events import FileSystemEventHandler as FileSystemWatchdogEventHandler
from tfw.networking.event_handlers import ServerUplinkConnector
from tfw.decorators import RateLimiter
from tfw.mixins import ObserverMixin
from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
from tfw.decorators.rate_limiter import RateLimiter
from tfw.mixins.observer_mixin import ObserverMixin
from tfw.config.logs import logging

6
lib/tfw/components/directory_monitoring_event_handler.py
View File

@ -3,10 +3,10 @@
from os.path import isdir, exists
from tfw import EventHandlerBase
from tfw.event_handler_base import EventHandlerBase
from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin
from tfw.components.directory_monitor import DirectoryMonitor
from tfw.config.logs import logging
from tfw.mixins import MonitorManagerMixin
from .directory_monitor import DirectoryMonitor
LOG = logging.getLogger(__name__)

87
lib/tfw/components/directory_snapshotting_event_handler.py Normal file
View File

@ -0,0 +1,87 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from os.path import join as joinpath
from os.path import basename
from os import makedirs
from datetime import datetime
from dateutil import parser as dateparser
from tfw.event_handler_base import EventHandlerBase
from tfw.components.snapshot_provider import SnapshotProvider
from tfw.config import TFWENV
from tfw.config.logs import logging
LOG = logging.getLogger(__name__)
class DirectorySnapshottingEventHandler(EventHandlerBase):
def __init__(self, key, directories, exclude_unix_patterns=None):
super().__init__(key)
self.snapshot_providers = {}
self._exclude_unix_patterns = exclude_unix_patterns
self.init_snapshot_providers(directories)
self.command_handlers = {
'take_snapshot': self.handle_take_snapshot,
'restore_snapshot': self.handle_restore_snapshot,
'exclude': self.handle_exclude
}
def init_snapshot_providers(self, directories):
for index, directory in enumerate(directories):
git_dir = self.init_git_dir(index, directory)
self.snapshot_providers[directory] = SnapshotProvider(
directory,
git_dir,
self._exclude_unix_patterns
)
@staticmethod
def init_git_dir(index, directory):
git_dir = joinpath(
TFWENV.SNAPSHOTS_DIR,
f'{basename(directory)}-{index}'
)
makedirs(git_dir, exist_ok=True)
return git_dir
def handle_event(self, message):
try:
data = message['data']
message['data'] = self.command_handlers[data['command']](data)
return message
except KeyError:
LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
def handle_take_snapshot(self, data):
LOG.debug('Taking snapshots of directories %s', self.snapshot_providers.keys())
for provider in self.snapshot_providers.values():
provider.take_snapshot()
return data
def handle_restore_snapshot(self, data):
date = dateparser.parse(
data.get(
'value',
datetime.now().isoformat()
)
)
LOG.debug(
'Restoring snapshots (@ %s) of directories %s',
date,
self.snapshot_providers.keys()
)
for provider in self.snapshot_providers.values():
provider.restore_snapshot(date)
return data
def handle_exclude(self, data):
exclude_unix_patterns = data['value']
if not isinstance(exclude_unix_patterns, list):
raise KeyError
for provider in self.snapshot_providers.values():
provider.exclude = exclude_unix_patterns
return data

46
lib/tfw/components/fsm_managing_event_handler.py
View File

@ -1,7 +1,7 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from tfw import EventHandlerBase
from tfw.event_handler_base import EventHandlerBase
from tfw.crypto import KeyManager, sign_message, verify_message
from tfw.config.logs import logging
@ -9,6 +9,20 @@ LOG = logging.getLogger(__name__)
class FSMManagingEventHandler(EventHandlerBase):
"""
EventHandler responsible for managing the state machine of
the framework (TFW FSM).
tfw.networking.TFWServer instances automatically send 'trigger'
commands to the event handler listening on the 'fsm' key,
which should be an instance of this event handler.
This event handler accepts messages that have a
data['command'] key specifying a command to be executed.
An 'fsm_update' message is broadcasted after every successful
command.
"""
def __init__(self, key, fsm_type, require_signature=False):
super().__init__(key)
self.fsm = fsm_type()
@ -25,7 +39,7 @@ class FSMManagingEventHandler(EventHandlerBase):
try:
message = self.command_handlers[message['data']['command']](message)
if message:
fsm_update_message = self._fsm_updater.generate_fsm_update()
fsm_update_message = self._fsm_updater.fsm_update
sign_message(self.auth_key, message)
sign_message(self.auth_key, fsm_update_message)
self.server_connector.broadcast(fsm_update_message)
@ -34,6 +48,12 @@ class FSMManagingEventHandler(EventHandlerBase):
LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
def handle_trigger(self, message):
"""
Attempts to step the FSM with the supplied trigger.
:param message: TFW message with a data field containing
the action to try triggering in data['value']
"""
trigger = message['data']['value']
if self._require_signature:
if not verify_message(self.auth_key, message):
@ -44,6 +64,9 @@ class FSMManagingEventHandler(EventHandlerBase):
return None
def handle_update(self, message):
"""
Does nothing, but triggers an 'fsm_update' message.
"""
# pylint: disable=no-self-use
return message
@ -52,23 +75,24 @@ class FSMUpdater:
def __init__(self, fsm):
self.fsm = fsm
def generate_fsm_update(self):
@property
def fsm_update(self):
return {
'key': 'fsm_update',
'data': self.get_fsm_state_and_transitions()
'data': self.fsm_update_data
}
def get_fsm_state_and_transitions(self):
state = self.fsm.state
@property
def fsm_update_data(self):
valid_transitions = [
{'trigger': trigger}
for trigger in self.fsm.get_triggers(self.fsm.state)
]
last_trigger = self.fsm.trigger_history[-1] if self.fsm.trigger_history else None
in_accepted_state = state in self.fsm.accepted_states
last_fsm_event = self.fsm.event_log[-1]
last_fsm_event['timestamp'] = last_fsm_event['timestamp'].isoformat()
return {
'current_state': state,
'current_state': self.fsm.state,
'valid_transitions': valid_transitions,
'last_trigger': last_trigger,
'in_accepted_state': in_accepted_state
'in_accepted_state': self.fsm.in_accepted_state,
'last_event': last_fsm_event
}

5
lib/tfw/components/history_monitor.py
View File

@ -8,8 +8,9 @@ from abc import ABC, abstractmethod
from watchdog.events import PatternMatchingEventHandler
from tfw.mixins import CallbackMixin, ObserverMixin
from tfw.decorators import RateLimiter
from tfw.mixins.callback_mixin import CallbackMixin
from tfw.mixins.observer_mixin import ObserverMixin
from tfw.decorators.rate_limiter import RateLimiter
class CallbackEventHandler(PatternMatchingEventHandler, ABC):

9
lib/tfw/components/ide_event_handler.py
View File

@ -6,10 +6,10 @@ from glob import glob
from fnmatch import fnmatchcase
from typing import Iterable
from tfw import EventHandlerBase
from tfw.mixins import MonitorManagerMixin
from tfw.event_handler_base import EventHandlerBase
from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin
from tfw.components.directory_monitor import DirectoryMonitor
from tfw.config.logs import logging
from .directory_monitor import DirectoryMonitor
LOG = logging.getLogger(__name__)
@ -157,7 +157,8 @@ class IdeEventHandler(EventHandlerBase, MonitorManagerMixin):
"""
Read the currently selected file.
:return dict: message with the contents of the file in data['content']
:return dict: TFW message data containing key 'content'
(contents of the selected file)
"""
try:
data['content'] = self.filemanager.file_contents

7
lib/tfw/components/log_monitor.py
View File

@ -6,9 +6,10 @@ from os.path import dirname
from watchdog.events import PatternMatchingEventHandler as PatternMatchingWatchdogEventHandler
from tfw.networking.event_handlers import ServerUplinkConnector
from tfw.decorators import RateLimiter
from tfw.mixins import ObserverMixin, SupervisorLogMixin
from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
from tfw.decorators.rate_limiter import RateLimiter
from tfw.mixins.observer_mixin import ObserverMixin
from tfw.mixins.supervisor_mixin import SupervisorLogMixin
class LogMonitor(ObserverMixin):

6
lib/tfw/components/log_monitoring_event_handler.py
View File

@ -1,10 +1,10 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from tfw import EventHandlerBase
from tfw.mixins import MonitorManagerMixin
from tfw.event_handler_base import EventHandlerBase
from tfw.mixins.monitor_manager_mixin import MonitorManagerMixin
from tfw.components.log_monitor import LogMonitor
from tfw.config.logs import logging
from .log_monitor import LogMonitor
LOG = logging.getLogger(__name__)

2
lib/tfw/components/pipe_io_event_handler.py
View File

@ -9,7 +9,7 @@ from secrets import token_urlsafe
from threading import Thread
from contextlib import suppress
from tfw import EventHandlerBase
from tfw.event_handler_base import EventHandlerBase
from tfw.config.logs import logging
from .pipe_io_server import PipeIOServer, terminate_process_on_failure

6
lib/tfw/components/process_managing_event_handler.py
View File

@ -3,10 +3,10 @@
from xmlrpc.client import Fault as SupervisorFault
from tfw import EventHandlerBase
from tfw.mixins import SupervisorMixin, SupervisorLogMixin
from tfw.event_handler_base import EventHandlerBase
from tfw.mixins.supervisor_mixin import SupervisorMixin, SupervisorLogMixin
from tfw.components.directory_monitor import with_monitor_paused
from tfw.config.logs import logging
from .directory_monitor import with_monitor_paused
LOG = logging.getLogger(__name__)

71
lib/tfw/components/snapshot_provider.py
View File

@ -2,15 +2,17 @@
# All Rights Reserved. See LICENSE file for details.
import re
from subprocess import run, CalledProcessError
from subprocess import run, CalledProcessError, PIPE
from getpass import getuser
from os.path import isdir
from datetime import datetime
from os.path import join as joinpath
from uuid import uuid4
from dateutil import parser as dateparser
class SnapshotProvider:
def __init__(self, directory, git_dir):
def __init__(self, directory, git_dir, exclude_unix_patterns=None):
self._classname = self.__class__.__name__
author = f'{getuser()} via TFW {self._classname}'
self.gitenv = {
@ -25,6 +27,8 @@ class SnapshotProvider:
self._init_repo()
self.__last_valid_branch = self._branch
if exclude_unix_patterns:
self.exclude = exclude_unix_patterns
def _init_repo(self):
self._check_environment()
@ -66,10 +70,14 @@ class SnapshotProvider:
'git', 'add',
'-A'
))
self._run((
'git', 'commit',
'-m', 'Snapshot'
))
try:
self._get_stdout((
'git', 'commit',
'-m', 'Snapshot'
))
except CalledProcessError as err:
if b'nothing to commit, working tree clean' not in err.output:
raise
def _check_head_not_detached(self):
if self._head_detached:
@ -87,7 +95,8 @@ class SnapshotProvider:
))
def _get_stdout(self, *args, **kwargs):
kwargs['capture_output'] = True
kwargs['stdout'] = PIPE
kwargs['stderr'] = PIPE
stdout_bytes = self._run(*args, **kwargs).stdout
return stdout_bytes.decode().rstrip('\n')
@ -98,13 +107,31 @@ class SnapshotProvider:
kwargs['env'] = self.gitenv
return run(*args, **kwargs)
@property
def exclude(self):
with open(self._exclude_path, 'r') as ofile:
return ofile.read()
@exclude.setter
def exclude(self, exclude_patterns):
with open(self._exclude_path, 'w') as ifile:
ifile.write('\n'.join(exclude_patterns))
@property
def _exclude_path(self):
return joinpath(
self.gitenv['GIT_DIR'],
'info',
'exclude'
)
def take_snapshot(self):
if self._head_detached:
self._checkout_new_branch_from_head()
self._snapshot()
def _checkout_new_branch_from_head(self):
branch_name = uuid4()
branch_name = str(uuid4())
self._run((
'git', 'branch',
branch_name
@ -119,16 +146,30 @@ class SnapshotProvider:
def restore_snapshot(self, date):
commit = self._get_commit_from_timestamp(date)
branch = self._last_valid_branch
if commit == self._latest_commit_on_branch(branch):
commit = branch
self._checkout(commit)
def _get_commit_from_timestamp(self, date):
return self._get_stdout((
commit = self._get_stdout((
'git', 'rev-list',
'--date=iso',
'-n', '1',
f'--before="{date.isoformat()}"',
self._last_valid_branch
))
if not commit:
commit = self._get_oldest_parent_of_head()
return commit
def _get_oldest_parent_of_head(self):
return self._get_stdout((
'git',
'rev-list',
'--max-parents=0',
'HEAD'
))
@property
def _last_valid_branch(self):
@ -136,6 +177,14 @@ class SnapshotProvider:
self.__last_valid_branch = self._branch
return self.__last_valid_branch
def _latest_commit_on_branch(self, branch):
return self._get_stdout((
'git', 'log',
'-n', '1',
'--pretty=format:%H',
branch
))
@property
def all_timelines(self):
return self._branches
@ -169,7 +218,7 @@ class SnapshotProvider:
commit_hash, timestamp = line.split('@')
commits.append({
'hash': commit_hash,
'timestamp': datetime.fromisoformat(timestamp)
'timestamp': dateparser.parse(timestamp)
})
return commits

5
lib/tfw/components/terminal_event_handler.py
View File

@ -1,11 +1,11 @@
# Copyright (C) 2018 Avatao.com Innovative Learning Kft.
# All Rights Reserved. See LICENSE file for details.
from tfw import EventHandlerBase
from tfw.event_handler_base import EventHandlerBase
from tfw.components.terminado_mini_server import TerminadoMiniServer
from tfw.config import TFWENV
from tfw.config.logs import logging
from tao.config import TAOENV
from .terminado_mini_server import TerminadoMiniServer
LOG = logging.getLogger(__name__)
@ -26,7 +26,6 @@ class TerminalEventHandler(EventHandlerBase):
:param monitor: tfw.components.HistoryMonitor instance to read command history from
"""
super().__init__(key)
self.working_directory = TFWENV.TERMINADO_DIR
self._historymonitor = monitor
bash_as_user_cmd = ['sudo', '-u', TAOENV.USER, 'bash']