Merge pull request #14 from avatao-content/connector-rm

Remove unnecessary webservice dependencies and replace TFWConnector with PipeIO

solvable/Dockerfile

@@ -1,11 +1,5 @@
 FROM eu.gcr.io/avatao-challengestore/tutorial-framework
 
-# Install webservice dependencies
-RUN pip3 install Flask==1.0        \
-                 SQLAlchemy==1.2.7 \
-                 passlib==1.7.1    \
-                 git+https://github.com/avatao-content/tfwconnector.git#subdirectory=python3
-
 # Define variables to use later
 ENV TFW_EHMAIN_DIR="${TFW_DIR}/builtin_event_handlers" \
     TFW_WEBSERVICE_DIR="/srv/webservice"               \

solvable/src/pipe_io_auxlib.py

@@ -0,0 +1,235 @@
+from json import dumps, loads
+
+from tfw.crypto import KeyManager, sign_message, verify_message
+from tfw.components import PipeIOEventHandlerBase
+from tfw.components.pipe_io_event_handler import DEFAULT_PERMISSIONS
+
+
+class SignMessagePipeIOEventHandler(PipeIOEventHandlerBase):
+    """
+    Signs a valid TFW message with HMAC.
+    Note that the running process needs root permissions in order to read
+    the authentication key.
+    When forwarding is true, it will send the signed message to the TFW
+    server before writing it into the output pipe.
+    """
+    
+    def __init__(
+            self, in_pipe_path, out_pipe_path,
+            permissions=DEFAULT_PERMISSIONS,
+            forwarding=True
+    ):
+        self.forwarding = forwarding
+        self.auth_key = KeyManager().auth_key
+        super().__init__(None, in_pipe_path, out_pipe_path, permissions)
+
+    def handle_event(self, message):
+        pass
+
+    def handle_pipe_event(self, message_bytes):
+        message = loads(message_bytes)
+        sign_message(self.auth_key, message)
+        if self.forwarding:
+            self.server_connector.send_message(message)
+        self.pipe_io.send_message(dumps(message).encode())
+
+
+class VerifyMessagePipeIOEventHandler(PipeIOEventHandlerBase):
+    """
+    Verifies a signed TFW message.
+    This pipe also needs root permissions. Send the serialized JSON object
+    to the pipe, then wait for its boolean response.
+    """
+
+    def __init__(self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS):
+        self.auth_key = KeyManager().auth_key
+        super().__init__(None, in_pipe_path, out_pipe_path, permissions)
+
+    def handle_event(self, message):
+        pass
+
+    def handle_pipe_event(self, message_bytes):
+        message = loads(message_bytes)
+        validity = verify_message(self.auth_key, message)
+        self.pipe_io.send_message(str(validity).lower().encode())
+
+
+class BotPipeIOEventHandler(PipeIOEventHandlerBase):
+    """
+    Sends bot messages to the frontend.
+    If you assign @originator, it will be the default message sender.
+    When you write a line to the pipe, it will be considered as a single
+    message and gets appended to the queue until an empty line is received,
+    which triggers forwarding the messages to the TFW server.
+    """
+
+    def __init__(
+            self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS,
+            originator='avataobot'
+    ):
+        self.queue = []
+        self.originator = originator
+        super().__init__(None, in_pipe_path, out_pipe_path, permissions)
+
+    def handle_event(self, message):
+        pass
+
+    def handle_pipe_event(self, message_bytes):
+        if message_bytes == b"":
+            if self.queue:
+                self.server_connector.send_message({
+                    'key': 'queueMessages',
+                    'data': {
+                        'messages': self.queue
+                    }
+                })
+                self.queue = []
+        else:
+            self.queue.append({
+                'originator': self.originator,
+                'message': message_bytes.decode().replace('\\n', '\n')
+            })
+
+
+class DeployPipeIOEventHandler(PipeIOEventHandlerBase):
+    """
+    Manages deployment in the IDE.
+    When you receive "deploy", then you have to answer with a "true" or
+    "false" depending whether you are satisfied with the result or not.
+    The @process parameter is the name of the supervised service.
+    """
+
+    # pylint: disable=too-many-arguments
+    def __init__(
+            self, in_pipe_path, out_pipe_path, process,
+            permissions=DEFAULT_PERMISSIONS
+    ):
+        self.expected = False
+        self.process = process
+
+        self.onsuccess = {
+            'key': 'processmanager',
+            'data': {
+                'process_name': process,
+                'command': 'restart'
+            }
+        }
+        self.onerror = {
+            'key': 'processmanager',
+            'data': {
+                'process_name': process,
+                'error': True
+            }
+        }
+
+        super().__init__('processmanager', in_pipe_path, out_pipe_path, permissions)
+
+    def handle_event(self, message):
+        if message == self.onsuccess:
+            self.expected = True
+            self.pipe_io.send_message(b'deploy')
+
+    def handle_pipe_event(self, message_bytes):
+        if not self.expected:
+            raise ValueError(
+                f'{self.pipe_io.in_pipe}: There is nothing to deploy.'
+            )
+
+        self.expected = False
+        if message_bytes == b'true':
+            self.server_connector.send_message(self.onsuccess)
+        elif message_bytes == b'false':
+            self.server_connector.send_message(self.onerror)
+        else:
+            raise ValueError(
+                f'{self.pipe_io.in_pipe}: Expected "true" or "false".'
+            )
+
+
+class IdePipeIOEventHandler(PipeIOEventHandlerBase):
+    """
+    Manipulates the content of the IDE.
+    You can observe a file, and when the user edits it, you will receive
+    the new contents where newlines are escaped as "\\n".
+    In order to overwrite the file, send an escaped text back to the pipe.
+    Since the pipe doesn't know if the file is selected initially in the IDE,
+    you have to provide this information by yourself with @selected,
+    but it will track it later on.
+    """
+
+    # pylint: disable=too-many-arguments
+    def __init__(
+            self, in_pipe_path, out_pipe_path, filename,
+            permissions=DEFAULT_PERMISSIONS,
+            selected=True
+    ):
+        self.selected = selected
+        self.filename = filename
+        super().__init__('ide', in_pipe_path, out_pipe_path, permissions)
+
+    def handle_event(self, message):
+        data = message['data']
+
+        if data['command'] == 'select':
+            self.selected = data['filename'] == self.filename
+        elif data['command'] == 'write' and self.selected:
+            clean = data['content'].replace('\n', '\\n')
+            self.pipe_io.send_message(clean.encode())
+
+    def handle_pipe_event(self, message_bytes):
+        if not self.selected:
+            self.server_connector.send_message({
+                'key': 'mirror',
+                'data': {
+                    'key': 'ide',
+                    'data': {
+                        'command': 'select',
+                        'filename': self.filename
+                    }
+                }
+            })
+
+        self.server_connector.send_message({
+            'key': 'mirror',
+            'data': {
+                'key': 'ide',
+                'data': {
+                    'command': 'write',
+                    'content': message_bytes.decode().replace('\\n', '\n')
+                }
+            }
+        })
+        self.server_connector.send_message({
+            'key': 'mirror',
+            'data': {
+                'key': 'ide',
+                'data': {
+                    'command': 'read'
+                }
+            }
+        })
+
+
+class FSMPipeIOEventHandler(PipeIOEventHandlerBase):
+    """
+    Handles FSM steps.
+    When the FSM enters the next state, you will receive a line containing
+    its name. To trigger a state change, send the name of the transition to
+    the pipe.
+    """
+
+    def __init__(self, in_pipe_path, out_pipe_path, permissions=DEFAULT_PERMISSIONS):
+        super().__init__(
+            ['fsm', 'fsm_update'],
+            in_pipe_path, out_pipe_path, permissions
+        )
+
+    def handle_event(self, message):
+        if 'current_state' in message['data']:
+            self.pipe_io.send_message(message['data']['current_state'].encode())
+
+    def handle_pipe_event(self, message_bytes):
+        self.server_connector.send_message({
+            'key': '',
+            'trigger': message_bytes.decode()
+        })

solvable/src/pipe_io_main.py

@@ -6,14 +6,67 @@ from tornado.ioloop import IOLoop
 from tfw.event_handlers import EventHandlerBase
 from tfw.components import PipeIOEventHandler
 
+from pipe_io_auxlib import (
+        SignMessagePipeIOEventHandler, VerifyMessagePipeIOEventHandler,
+        BotPipeIOEventHandler,
+        DeployPipeIOEventHandler, IdePipeIOEventHandler,
+        FSMPipeIOEventHandler
+)
+
 logging.basicConfig(level=logging.DEBUG)
 
 
 if __name__ == '__main__':
-    pipe_io = PipeIOEventHandler(
+    """
+    Creates general purpose pipes.
+    The first parameter associates the receiving pipe with a key, which is
+    an empty string in this case. It has a special meaning, you can
+    subscribe to every kind of message with this key.
+    If you wish to filter incoming data, specify a single or more keys in
+    a list, eg.: processmanager, ide, key...
+    You can send/receive JSON messages to/from the TFW server as any user,
+    because we gave read+write permissions, without that parameter, only
+    the owner has access to the pipes.
+    """
+    json_pipe = PipeIOEventHandler(
         '',
-        '/tmp/tfw_send',
-        '/tmp/tfw_recv'
+        '/tmp/tfw_json_send',
+        '/tmp/tfw_json_recv'
+    )
+
+    sign_pipe = SignMessagePipeIOEventHandler(
+        '/tmp/tfw_sign_send',
+        '/tmp/tfw_sign_recv',
+        forwarding=True
+    )
+
+    verify_pipe = VerifyMessagePipeIOEventHandler(
+        '/tmp/tfw_verify_send',
+        '/tmp/tfw_verify_recv'
+    )
+
+    bot_pipe = BotPipeIOEventHandler(
+        '/tmp/tfw_bot_send',
+        '/tmp/tfw_bot_recv',
+        permissions=0o666
+    )
+
+    deploy_pipe = DeployPipeIOEventHandler(
+        '/tmp/tfw_deploy_send',
+        '/tmp/tfw_deploy_recv',
+        'webservice'
+    )
+
+    ide_pipe = IdePipeIOEventHandler(
+        '/tmp/tfw_ide_send',
+        '/tmp/tfw_ide_recv',
+        'user_ops.py',
+        selected=True
+    )
+
+    fsm_pipe = FSMPipeIOEventHandler(
+        '/tmp/tfw_fsm_send',
+        '/tmp/tfw_fsm_recv'
     )
 
     event_handlers = EventHandlerBase.get_local_instances()

solvable/src/test_fsm.yml

@@ -4,25 +4,21 @@ states:
     -   name: '0'
     -   name: '1'
         on_enter: |
-            python3 -c "from tfwconnector import MessageSender; MessageSender().send('FSM', 'Entered state 1!')"
+            printf "Entered state 1!\n\n" > /tmp/tfw_bot_send
     -   name: '2'
         on_enter: |
             file=/home/user/workdir/cat.txt
             echo "As you can see it is possible to execute arbitrary shell commands here." >> $file
-            python3 -c \
-            "
-            from tfwconnector import MessageSender
-            MessageSender().send('FSM', 'Entered state 2! Written stuff to $file')
-            "
+            printf "Entered state 2! Written stuff to $file\n\n" > /tmp/tfw_bot_send
     -   name: '3'
         on_enter: |
-            python3 -c "from tfwconnector import MessageSender; MessageSender().send('FSM', 'Entered state 3!')"
+            printf "Entered state 3!\n\n" > /tmp/tfw_bot_send
     -   name: '4'
         on_enter: |
-            python3 -c "from tfwconnector import MessageSender; MessageSender().send('FSM', 'Entered state 4!')"
+            printf "Entered state 4!\n\n" > /tmp/tfw_bot_send
     -   name: '5'
         on_enter: |
-            python3 -c "from tfwconnector import MessageSender; MessageSender().send('FSM', 'Entered state 5!')"
+            printf "Entered state 5!\n\n" > /tmp/tfw_bot_send
 transitions:
     -   trigger: step_1
         source: '0'

solvable/src/webservice/crypto.py

@@ -1,11 +1,24 @@
-from passlib.hash import pbkdf2_sha256
+from os import urandom
+from hashlib import scrypt
 
 
 class PasswordHasher:
-    @staticmethod
-    def hash(password):
-        return pbkdf2_sha256.hash(password)
+    n = 16384
+    r = 8
+    p = 1
+    dklen = 32
 
-    @staticmethod
-    def verify(password, hashdigest):
-        return pbkdf2_sha256.verify(password, hashdigest)
+    @classmethod
+    def hash(cls, password):
+        salt = urandom(32)
+        return cls.scrypt(password, salt).hex() + salt.hex()
+
+    @classmethod
+    def verify(cls, password, salted_hash):
+        salt = bytes.fromhex(salted_hash)[32:]
+        hashdigest = bytes.fromhex(salted_hash)[:32]
+        return cls.scrypt(password, salt) == hashdigest
+
+    @classmethod
+    def scrypt(cls, password, salt):
+        return scrypt(password.encode(), salt=salt, n=cls.n, r=cls.r, p=cls.p, dklen=cls.dklen)

solvable/src/webservice/pipe_io.py

@@ -0,0 +1,69 @@
+from typing import Callable
+
+
+class PipeReader:
+    def __init__(self, pipe_path):
+        self._pipe = open(pipe_path, 'rb')
+        self._message_handler = lambda msg: None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type_, value, traceback):
+        self.close()
+
+    def close(self):
+        self._pipe.close()
+
+    @property
+    def message_handler(self):
+        return self._message_handler
+
+    @message_handler.setter
+    def message_handler(self, value):
+        if not isinstance(value, Callable):
+            raise ValueError("message_handler must be callable!")
+        self._message_handler = value
+
+    def run(self):
+        msg = self.recv_message()
+        while msg:
+            self._message_handler(msg)
+            msg = self.recv_message()
+
+    def recv_message(self):
+        return self._pipe.readline()[:-1]
+
+
+class PipeWriter:
+    def __init__(self, pipe_path):
+        self._pipe = open(pipe_path, 'wb')
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type_, value, traceback):
+        self.close()
+
+    def close(self):
+        self._pipe.close()
+
+    def send_message(self, message):
+        self._pipe.write(message + b'\n')
+        self._pipe.flush()
+
+
+class PipeIO:
+    def __init__(self, in_pipe_path, out_pipe_path):
+        self.reader = PipeReader(in_pipe_path)
+        self.writer = PipeWriter(out_pipe_path)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type_, value, traceback):
+        self.close()
+
+    def close(self):
+        self.reader.close()
+        self.writer.close()

solvable/src/webservice/user_ops.py

@@ -1,7 +1,6 @@
-from functools import partial
-
-from tfwconnector import MessageSender
+from json import dumps
 
+from pipe_io import PipeWriter
 from crypto import PasswordHasher
 from model import User
 from errors import InvalidCredentialsError, UserExistsError
@@ -12,11 +11,8 @@ class UserOps:
         self.username = username
         self.password = password
         self.db_session = db_session
-        self.message_sender = MessageSender()
-        self.log = partial(
-            self.message_sender.send,
-            'Authenticator'
-        )
+        self.pipe = PipeWriter('/tmp/tfw_bot_send')
+        self.log = lambda message: self.pipe.send_message(message.encode()+b"\n")
 
     def authenticate(self):
         """