Merge branch 'controller'

Dockerfile

@@ -39,29 +39,28 @@ COPY src/frontend /data/
 RUN yarn install --frozen-lockfile
 RUN yarn build --no-progress
 
-ENV TFW_WEB_PORT=4242
+ENV TFW_PUBLIC_PORT=8888          \
-ENV TFW_LOGIN_APP_PORT=6666
+    TFW_WEB_PORT=4242             \
-ENV TFW_TERMINADO_PORT=7878
+    TFW_LOGIN_APP_PORT=6666       \
-ENV TFW_SUPERVISOR_HTTP_PORT=9001
+    TFW_TERMINADO_PORT=7878       \
-ENV TFW_PUBLIC_PORT=8888
+    TFW_SUPERVISOR_HTTP_PORT=9001 \
-ENV TFW_PUBLISHER_PORT=7654
+    TFW_CONTROLLER_PORT=7777      \
-ENV TFW_RECEIVER_PORT=8765
+    TFW_CRP_LISTENER_PORT=5555    \
-EXPOSE ${TFW_PUBLIC_PORT}
+    TFW_PUBLISHER_PORT=7654       \
+    TFW_RECEIVER_PORT=8765
 
-ENV TFW_SUPERVISOR_HTTP_URI="http://localhost:${TFW_SUPERVISOR_HTTP_PORT}"
+EXPOSE ${TFW_PUBLIC_PORT} ${TFW_CRP_LISTENER_PORT}
-ENV TFW_EVENT_HANDLERS_DIR="/opt/event_handlers"
+
-ENV TFW_APP_DIR="/srv/app"
+ENV PYTHONPATH="/usr/local/lib/"                                           \
-ENV TFW_FRONTEND_DIR="/srv/frontend"
+    TFW_SUPERVISOR_HTTP_URI="http://localhost:${TFW_SUPERVISOR_HTTP_PORT}" \
-ENV TFW_LOGIN_APP_DIR="/tmp/source_code_server"
+    TFW_SUPERVISORD_CONF="/etc/supervisor/supervisord.conf"                \
-ENV TFW_TERMINADO_DIR="/tmp/terminado_server"
+    TFW_SUPERVISORD_COMPONENTS="/etc/supervisor/conf"                      \
-ENV TFW_WEBIDE_WD="/home/${AVATAO_USER}/workdir"
+    TFW_NGINX_CONF="/etc/nginx/sites-enabled/default"                      \
-ENV TFW_TERMINADO_WD=${TFW_WEBIDE_WD}
+    TFW_NGINX_COMPONENTS="/etc/nginx/components"                           \
-ENV TFW_LIB_DIR="/usr/local/lib/"
+    TFW_LIB_DIR="/usr/local/lib/"                                          \
-ENV TFW_SUPERVISORD_CONF="/etc/supervisor/supervisord.conf"
+    TFW_CONTROLLER_DIR="/srv/controller"                                   \
-ENV TFW_SUPERVISORD_COMPONENTS="/etc/supervisor/conf"
+    TFW_TERMINADO_DIR="/tmp/terminado_server"                              \
-ENV TFW_NGINX_CONF="/etc/nginx/sites-enabled/default"
+    TFW_FRONTEND_DIR="/srv/frontend"
-ENV TFW_NGINX_COMPONENTS="/etc/nginx/components"
-ENV PYTHONPATH=${TFW_LIB_DIR}
 
 COPY nginx/nginx.conf ${TFW_NGINX_CONF}
 COPY nginx/components/ ${TFW_NGINX_COMPONENTS}
@@ -71,17 +70,26 @@ RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\
         envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f ;\
     done
 
-COPY lib ${TFW_LIB_DIR}
 COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF}
 COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS}
-COPY src/app ${TFW_APP_DIR}
+COPY lib ${TFW_LIB_DIR}
-COPY src/event_handlers ${TFW_EVENT_HANDLERS_DIR}
+COPY src/controller ${TFW_CONTROLLER_DIR}
+COPY lib/tfw/components/terminado_mini_server.py ${TFW_TERMINADO_DIR}/
+
+### TFW internals ^ ### DEMO v ###############################################################
+
+ENV TFW_APP_DIR="/srv/app"                          \
+    TFW_LOGIN_APP_DIR="/tmp/source_code_server"     \
+    TFW_WEBIDE_WD="/home/${AVATAO_USER}/workdir"    \
+    TFW_TERMINADO_WD="/home/${AVATAO_USER}/workdir"
 
 RUN mv /data/dist ${TFW_FRONTEND_DIR}
 
-COPY src/event_handlers/source_code_server/server.py ${TFW_LOGIN_APP_DIR}/
+COPY src/demo ${TFW_APP_DIR}/
-COPY src/event_handlers/source_code_server/users.db ${TFW_LOGIN_APP_DIR}/
+
-COPY src/event_handlers/source_code_server/login_component.py ${TFW_WEBIDE_WD}/
+COPY src/demo/source_code_server/server.py ${TFW_LOGIN_APP_DIR}/
+COPY src/demo/source_code_server/users.db ${TFW_LOGIN_APP_DIR}/
+COPY src/demo/source_code_server/login_component.py ${TFW_WEBIDE_WD}/
 
 RUN chown -R ${AVATAO_USER} ${TFW_WEBIDE_WD} && chmod -R 755 ${TFW_WEBIDE_WD}
 

lib/envvars.py

@@ -0,0 +1,9 @@
+from collections import namedtuple
+from os import environ
+
+
+def generate_namedtuple_from_prefixed_envvars(prefix: str, tuple_name: str):
+    envvars = {envvar.replace(prefix, '', 1): environ.get(envvar)
+               for envvar in environ.keys()
+               if envvar.startswith(prefix)}
+    return namedtuple(tuple_name, envvars)(**envvars)

lib/tao/config/__init__.py

@@ -0,0 +1 @@
+from .envvars import *

lib/tao/config/envvars.py

@@ -0,0 +1,3 @@
+from envvars import generate_namedtuple_from_prefixed_envvars
+
+taoenv = generate_namedtuple_from_prefixed_envvars('AVATAO_', 'taoenvtuple')

lib/tfw/components/terminado_event_handler.py

@@ -1,5 +1,3 @@
-from shutil import rmtree, copytree
-
 from tfw.event_handler_base import EventHandlerBase
 from tfw.util import SupervisorMixin
 from tfw.config import tfwenv
@@ -12,11 +10,6 @@ class TerminadoEventHandler(EventHandlerBase, SupervisorMixin):
         super().__init__(anchor)
         self.working_directory = tfwenv.TERMINADO_DIR
         self.process_name = process_name
-        self.setup_terminado_server()
-
-    def setup_terminado_server(self):
-        rmtree(self.working_directory, ignore_errors=True)
-        copytree('terminado_server/', self.working_directory)
         self.start_process()
 
     def handle_event(self, anchor, data_json):

lib/tfw/config/envvars.py

@@ -1,8 +1,3 @@
-from os import environ
+from envvars import generate_namedtuple_from_prefixed_envvars
-from collections import namedtuple
 
-TFW_PREFIX = 'TFW_'
+tfwenv = generate_namedtuple_from_prefixed_envvars('TFW_', 'tfwenvtuple')
-tfwenvvars = {envvar.replace(TFW_PREFIX, '', 1): environ.get(envvar)
-              for envvar in environ.keys()
-              if envvar.startswith(TFW_PREFIX)}
-tfwenv = namedtuple('tfwenvtuple', tfwenvvars)(**tfwenvvars)

lib/tfw/fsm_base.py

@@ -1,11 +1,14 @@
+from typing import List
+
 from transitions import Machine
 
 
 class FSMBase:
     states, transitions = [], []
 
-    def __init__(self, initial: str = None):
+    def __init__(self, initial: str = None, accepted_states: List[str] = None):
         self.message_handlers = []
+        self.accepted_states = accepted_states or [self.states[-1]]
         self.machine = Machine(model=self,
                                states=self.states,
                                transitions=self.transitions,
@@ -24,3 +27,6 @@ class FSMBase:
 
     def unsubscribe_message_handler(self, msghandler):
         self.message_handlers.remove(msghandler)
+
+    def is_solved(self):
+        return self.state in self.accepted_states

lib/tfw/networking/controller_connector.py

@@ -0,0 +1,18 @@
+import zmq
+from zmq.eventloop import ioloop
+from zmq.eventloop.zmqstream import ZMQStream
+
+from tfw.config import tfwenv
+from tfw.util import ZMQConnectorBase
+
+ioloop.install()
+
+
+class ControllerConnector(ZMQConnectorBase):
+    def __init__(self, zmq_context=None):
+        super(ControllerConnector, self).__init__(zmq_context)
+        self._zmq_rep_socket = self._zmq_context.socket(zmq.REP)
+        self._zmq_rep_socket.connect('tcp://localhost:{}'.format(tfwenv.CONTROLLER_PORT))
+        self._zmq_rep_stream = ZMQStream(self._zmq_rep_socket)
+
+        self.register_callback = self._zmq_rep_stream.on_recv_stream

lib/tfw/networking/serialization.py

@@ -13,9 +13,9 @@ def decode_if_needed(value):
     return value
 
 
-def serialize_all(anchor, message):
+def serialize_all(key, data):
-    return [encode_if_needed(a) for a in (anchor, json.dumps(message))]
+    return [encode_if_needed(frame) for frame in (key, json.dumps(data))]
 
 
-def deserialize_all(anchor, message):
+def deserialize_all(key, data):
-    return decode_if_needed(anchor), json.loads(message)
+    return decode_if_needed(key), json.loads(data)

lib/tfw/networking/server/tfw_server.py

@@ -0,0 +1,33 @@
+from tornado.web import Application
+
+from tfw.networking.controller_connector import ControllerConnector
+from tfw.networking.serialization import deserialize_all, serialize_all
+from tfw.networking.server.zmq_websocket_handler import FSMManagingSocketHandler
+
+
+class TFWServer:
+    def __init__(self, fsm_type):
+        self._fsm = fsm_type()
+        self.application = Application(
+            [(r'/ws', FSMManagingSocketHandler, {'fsm': self.fsm})],
+            autoreload=True
+        )
+        self.controller_connector = ControllerConnector()
+        self.controller_connector.register_callback(self.zmq_callback)
+
+    @property
+    def fsm(self):
+        return self._fsm
+
+    def zmq_callback(self, stream, msg_parts):
+        key, data = deserialize_all(*msg_parts)
+        if key == 'test':
+            stream.send_multipart(serialize_all(key, 'OK'))
+        if key == 'solution_check':
+            stream.send_multipart(serialize_all(key, {
+                'solved': self.fsm.is_solved(),
+                'message': 'solved' if self.fsm.is_solved() else 'not solved'
+            }))
+
+    def listen(self, port):
+        self.application.listen(port)

lib/tfw/networking/server/zmq_websocket_handler.py

@@ -26,15 +26,9 @@ class ZMQWebSocketHandler(WebSocketHandler):
         log.debug('Received on WebSocket: {}'.format(message))
         self.send_message(*self.make_response(json.loads(message)))
 
-    def make_response(self, message):
-        raise NotImplementedError
-
     def send_message(self, message: dict, anchor: str = None):
         self._event_handler_connector.send_message(message, anchor)
 
-    def on_close(self):
-        pass
-
     # much secure, very cors, wow
     def check_origin(self, origin):
         return True

lib/tfw/networking/solvable_connector.py

@@ -0,0 +1,21 @@
+import zmq
+from zmq.eventloop import ioloop
+
+from tfw.config import tfwenv
+from tfw.util import ZMQConnectorBase
+from tfw.networking.serialization import serialize_all, deserialize_all
+
+ioloop.install()
+
+
+class SolvableConnector(ZMQConnectorBase):
+    def __init__(self, zmq_context=None):
+        super(SolvableConnector, self).__init__(zmq_context)
+        self._zmq_req_socket = self._zmq_context.socket(zmq.REQ)
+        self._zmq_req_socket.bind('tcp://*:{}'.format(tfwenv.CONTROLLER_PORT))
+
+    def send(self, key, message):
+        self._zmq_req_socket.send_multipart(serialize_all(key, message))
+
+    def recv(self):
+        return deserialize_all(*self._zmq_req_socket.recv_multipart())

src/controller/app.py

@@ -0,0 +1,23 @@
+from tornado.ioloop import IOLoop
+from tornado.web import Application
+
+from tao.config import taoenv
+from tfw.config import tfwenv
+from handlers import SolutionCheckHandler, TestHandler
+from tfw.networking.solvable_connector import SolvableConnector
+
+from tfw.config.logs import logging
+log = logging.getLogger(__name__)
+
+if __name__ == '__main__':
+    solvable_connector = SolvableConnector()
+    routes = [
+        (r'/{secret}/?'.format(secret=taoenv.SECRET), SolutionCheckHandler, {'solvable_connector': solvable_connector}),
+        (r'/{secret}/test/?'.format(secret=taoenv.SECRET), TestHandler, {'solvable_connector': solvable_connector})
+    ]
+    app = Application(
+        routes
+    )
+    app.listen(tfwenv.CRP_LISTENER_PORT)
+    log.debug('Controller listening on {}'.format(tfwenv.CRP_LISTENER_PORT))
+    IOLoop.instance().start()

src/controller/handlers/__init__.py

@@ -0,0 +1,2 @@
+from .solution_check_handler import SolutionCheckHandler
+from .test_handler import TestHandler

src/controller/handlers/solution_check_handler.py

@@ -0,0 +1,15 @@
+from tornado.web import RequestHandler
+
+from tfw.config.logs import logging
+log = logging.getLogger(__name__)
+
+class SolutionCheckHandler(RequestHandler):
+    def initialize(self, solvable_connector):
+        self.solvable_connector = solvable_connector
+
+    def get(self):
+        log.debug('Sending request to solvable')
+        self.solvable_connector.send('solution_check', {})
+        resp_key, resp_data = self.solvable_connector.recv()
+        log.debug('Received answer from solvable')
+        self.write(resp_data)

src/controller/handlers/test_handler.py

@@ -0,0 +1,11 @@
+from tornado.web import RequestHandler
+
+
+class TestHandler(RequestHandler):
+    def initialize(self, solvable_connector):
+        self.solvable_connector = solvable_connector
+
+    def get(self):
+        self.solvable_connector.send('test', {})
+        resp_key, resp_data = self.solvable_connector.recv()
+        self.write(resp_data)

src/demo/app.py

@@ -1,27 +1,18 @@
 import sys
 import tornado
 import zmq
-from tornado.web import Application
 from tornado.ioloop import IOLoop
 
 from sql_injection_fsm import SQLInjectionFSM
+from tfw.networking.server.tfw_server import TFWServer
 from tfw.config import tfwenv
 from tfw.config.logs import logging
 log = logging.getLogger(__name__)
-from tfw.networking.server.zmq_websocket_handler import FSMManagingSocketHandler
 
 
 if __name__ == '__main__':
-    fsm = SQLInjectionFSM()
+    server = TFWServer(SQLInjectionFSM)
-    routes = [
+    server.listen(tfwenv.WEB_PORT)
-        (r'/ws', FSMManagingSocketHandler, {'fsm': fsm}),
-    ]
-    application = Application(
-        routes,
-        autoreload=True
-    )
-
-    application.listen(tfwenv.WEB_PORT)
     log.debug('Python version: {}'.format(sys.version[:5]))
     log.debug('Tornado version: {}'.format(tornado.version))
     log.debug('ZeroMQ version: {}'.format(zmq.zmq_version()))

src/demo/event_handler_main.py

@@ -1,5 +1,5 @@
-from source_code_event_handler import SourceCodeEventHandler
+from tfw.components.source_code_event_handler import SourceCodeEventHandler
-from terminado_event_handler import TerminadoEventHandler
+from tfw.components.terminado_event_handler import TerminadoEventHandler
 from tornado.ioloop import IOLoop
 
 from tfw.config import tfwenv

supervisor/components/controller.conf

@@ -0,0 +1,3 @@
+[program:controller]
+directory=%(ENV_TFW_CONTROLLER_DIR)s
+command=env python app.py

supervisor/components/event_handlers.conf

@@ -1,3 +1,3 @@
 [program:event_handler_main]
-directory=%(ENV_TFW_EVENT_HANDLERS_DIR)s
+directory=%(ENV_TFW_APP_DIR)s
 command=env python event_handler_main.py

supervisor/components/terminado.conf

@@ -1,4 +1,4 @@
 [program:terminado]
 directory=%(ENV_TFW_TERMINADO_DIR)s
-command=env python server.py
+command=env python terminado_mini_server.py
 autostart=false