diff --git a/Dockerfile b/Dockerfile index 195a4af..4147105 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,19 @@ FROM avatao/ubuntu:16.04 -RUN curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - &&\ - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - &&\ +RUN curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - &&\ + curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - &&\ echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list &&\ - apt-get update &&\ + apt-get update &&\ apt-get install -y --no-install-recommends \ - nodejs \ - yarn \ - supervisor \ - libzmq5 \ - nginx \ - gettext-base \ - libbz2-dev \ - libreadline-dev \ - libsqlite3-dev &&\ + nodejs \ + yarn \ + supervisor \ + libzmq5 \ + nginx \ + gettext-base \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev &&\ rm -rf /var/lib/apt/lists/* USER ${AVATAO_USER} @@ -23,14 +23,14 @@ COPY requirements.txt /tmp ARG PYTHON_VERSION="3.6.4" ARG PYENV_INSTALLER_URL="https://raw.githubusercontent.com/pyenv/pyenv-installer/78cfd4d/bin/pyenv-installer" ARG PYENV_INSTALLER_HASH=9509348b828f0564358fff456f7f693dd9ace351dc3f240854d7685ad8a8e1dd -RUN curl -fSL -o pyenv-installer ${PYENV_INSTALLER_URL} &&\ +RUN curl -fSL -o pyenv-installer ${PYENV_INSTALLER_URL} &&\ echo "${PYENV_INSTALLER_HASH} *pyenv-installer" | sha256sum -c - &&\ - bash pyenv-installer &&\ - rm pyenv-installer &&\ - echo "source $HOME/.pyenvrc" >> .bashrc &&\ - . $HOME/.pyenvrc &&\ - pyenv install ${PYTHON_VERSION} &&\ - pyenv global ${PYTHON_VERSION} &&\ + bash pyenv-installer &&\ + rm pyenv-installer &&\ + echo "source $HOME/.pyenvrc" >> .bashrc &&\ + . $HOME/.pyenvrc &&\ + pyenv install ${PYTHON_VERSION} &&\ + pyenv global ${PYTHON_VERSION} &&\ pip install -r /tmp/requirements.txt USER root @@ -39,49 +39,57 @@ COPY src/frontend /data/ RUN yarn install --frozen-lockfile RUN yarn build --no-progress -ENV TFW_WEB_PORT=4242 -ENV TFW_LOGIN_APP_PORT=6666 -ENV TFW_TERMINADO_PORT=7878 -ENV TFW_SUPERVISOR_HTTP_PORT=9001 -ENV TFW_PUBLIC_PORT=8888 -ENV TFW_PUBLISHER_PORT=7654 -ENV TFW_RECEIVER_PORT=8765 -EXPOSE ${TFW_PUBLIC_PORT} +ENV TFW_PUBLIC_PORT=8888 \ + TFW_WEB_PORT=4242 \ + TFW_LOGIN_APP_PORT=6666 \ + TFW_TERMINADO_PORT=7878 \ + TFW_SUPERVISOR_HTTP_PORT=9001 \ + TFW_CONTROLLER_PORT=7777 \ + TFW_CRP_LISTENER_PORT=5555 \ + TFW_PUBLISHER_PORT=7654 \ + TFW_RECEIVER_PORT=8765 -ENV TFW_SUPERVISOR_HTTP_URI="http://localhost:${TFW_SUPERVISOR_HTTP_PORT}" -ENV TFW_EVENT_HANDLERS_DIR="/opt/event_handlers" -ENV TFW_APP_DIR="/srv/app" -ENV TFW_FRONTEND_DIR="/srv/frontend" -ENV TFW_LOGIN_APP_DIR="/tmp/source_code_server" -ENV TFW_TERMINADO_DIR="/tmp/terminado_server" -ENV TFW_WEBIDE_WD="/home/${AVATAO_USER}/workdir" -ENV TFW_TERMINADO_WD=${TFW_WEBIDE_WD} -ENV TFW_LIB_DIR="/usr/local/lib/" -ENV TFW_SUPERVISORD_CONF="/etc/supervisor/supervisord.conf" -ENV TFW_SUPERVISORD_COMPONENTS="/etc/supervisor/conf" -ENV TFW_NGINX_CONF="/etc/nginx/sites-enabled/default" -ENV TFW_NGINX_COMPONENTS="/etc/nginx/components" -ENV PYTHONPATH=${TFW_LIB_DIR} +EXPOSE ${TFW_PUBLIC_PORT} ${TFW_CRP_LISTENER_PORT} + +ENV PYTHONPATH="/usr/local/lib/" \ + TFW_SUPERVISOR_HTTP_URI="http://localhost:${TFW_SUPERVISOR_HTTP_PORT}" \ + TFW_SUPERVISORD_CONF="/etc/supervisor/supervisord.conf" \ + TFW_SUPERVISORD_COMPONENTS="/etc/supervisor/conf" \ + TFW_NGINX_CONF="/etc/nginx/sites-enabled/default" \ + TFW_NGINX_COMPONENTS="/etc/nginx/components" \ + TFW_LIB_DIR="/usr/local/lib/" \ + TFW_CONTROLLER_DIR="/srv/controller" \ + TFW_TERMINADO_DIR="/tmp/terminado_server" \ + TFW_FRONTEND_DIR="/srv/frontend" COPY nginx/nginx.conf ${TFW_NGINX_CONF} COPY nginx/components/ ${TFW_NGINX_COMPONENTS} -RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\ - sed -i 's#pid /run/nginx.pid;#pid /tmp/nginx.pid;#g' /etc/nginx/nginx.conf &&\ - for f in "${TFW_NGINX_CONF}" ${TFW_NGINX_COMPONENTS}/*.conf; do \ +RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\ + sed -i 's#pid /run/nginx.pid;#pid /tmp/nginx.pid;#g' /etc/nginx/nginx.conf &&\ + for f in "${TFW_NGINX_CONF}" ${TFW_NGINX_COMPONENTS}/*.conf; do \ envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f ;\ done -COPY lib ${TFW_LIB_DIR} COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF} COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS} -COPY src/app ${TFW_APP_DIR} -COPY src/event_handlers ${TFW_EVENT_HANDLERS_DIR} +COPY lib ${TFW_LIB_DIR} +COPY src/controller ${TFW_CONTROLLER_DIR} +COPY lib/tfw/components/terminado_mini_server.py ${TFW_TERMINADO_DIR}/ + +### TFW internals ^ ### DEMO v ############################################################### + +ENV TFW_APP_DIR="/srv/app" \ + TFW_LOGIN_APP_DIR="/tmp/source_code_server" \ + TFW_WEBIDE_WD="/home/${AVATAO_USER}/workdir" \ + TFW_TERMINADO_WD="/home/${AVATAO_USER}/workdir" RUN mv /data/dist ${TFW_FRONTEND_DIR} -COPY src/event_handlers/source_code_server/server.py ${TFW_LOGIN_APP_DIR}/ -COPY src/event_handlers/source_code_server/users.db ${TFW_LOGIN_APP_DIR}/ -COPY src/event_handlers/source_code_server/login_component.py ${TFW_WEBIDE_WD}/ +COPY src/demo ${TFW_APP_DIR}/ + +COPY src/demo/source_code_server/server.py ${TFW_LOGIN_APP_DIR}/ +COPY src/demo/source_code_server/users.db ${TFW_LOGIN_APP_DIR}/ +COPY src/demo/source_code_server/login_component.py ${TFW_WEBIDE_WD}/ RUN chown -R ${AVATAO_USER} ${TFW_WEBIDE_WD} && chmod -R 755 ${TFW_WEBIDE_WD} diff --git a/lib/envvars.py b/lib/envvars.py new file mode 100644 index 0000000..aeb2712 --- /dev/null +++ b/lib/envvars.py @@ -0,0 +1,9 @@ +from collections import namedtuple +from os import environ + + +def generate_namedtuple_from_prefixed_envvars(prefix: str, tuple_name: str): + envvars = {envvar.replace(prefix, '', 1): environ.get(envvar) + for envvar in environ.keys() + if envvar.startswith(prefix)} + return namedtuple(tuple_name, envvars)(**envvars) diff --git a/lib/tao/__init__.py b/lib/tao/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/tao/config/__init__.py b/lib/tao/config/__init__.py new file mode 100644 index 0000000..d3319d7 --- /dev/null +++ b/lib/tao/config/__init__.py @@ -0,0 +1 @@ +from .envvars import * diff --git a/lib/tao/config/envvars.py b/lib/tao/config/envvars.py new file mode 100644 index 0000000..6262ead --- /dev/null +++ b/lib/tao/config/envvars.py @@ -0,0 +1,3 @@ +from envvars import generate_namedtuple_from_prefixed_envvars + +taoenv = generate_namedtuple_from_prefixed_envvars('AVATAO_', 'taoenvtuple') diff --git a/src/event_handlers/source_code_event_handler.py b/lib/tfw/components/source_code_event_handler.py similarity index 100% rename from src/event_handlers/source_code_event_handler.py rename to lib/tfw/components/source_code_event_handler.py diff --git a/src/event_handlers/terminado_event_handler.py b/lib/tfw/components/terminado_event_handler.py similarity index 72% rename from src/event_handlers/terminado_event_handler.py rename to lib/tfw/components/terminado_event_handler.py index 9852247..4222e05 100644 --- a/src/event_handlers/terminado_event_handler.py +++ b/lib/tfw/components/terminado_event_handler.py @@ -1,5 +1,3 @@ -from shutil import rmtree, copytree - from tfw.event_handler_base import EventHandlerBase from tfw.util import SupervisorMixin from tfw.config import tfwenv @@ -12,11 +10,6 @@ class TerminadoEventHandler(EventHandlerBase, SupervisorMixin): super().__init__(anchor) self.working_directory = tfwenv.TERMINADO_DIR self.process_name = process_name - self.setup_terminado_server() - - def setup_terminado_server(self): - rmtree(self.working_directory, ignore_errors=True) - copytree('terminado_server/', self.working_directory) self.start_process() def handle_event(self, anchor, data_json): diff --git a/src/event_handlers/terminado_server/server.py b/lib/tfw/components/terminado_mini_server.py similarity index 100% rename from src/event_handlers/terminado_server/server.py rename to lib/tfw/components/terminado_mini_server.py diff --git a/lib/tfw/config/envvars.py b/lib/tfw/config/envvars.py index b9fa997..d94be2b 100644 --- a/lib/tfw/config/envvars.py +++ b/lib/tfw/config/envvars.py @@ -1,8 +1,3 @@ -from os import environ -from collections import namedtuple +from envvars import generate_namedtuple_from_prefixed_envvars -TFW_PREFIX = 'TFW_' -tfwenvvars = {envvar.replace(TFW_PREFIX, '', 1): environ.get(envvar) - for envvar in environ.keys() - if envvar.startswith(TFW_PREFIX)} -tfwenv = namedtuple('tfwenvtuple', tfwenvvars)(**tfwenvvars) +tfwenv = generate_namedtuple_from_prefixed_envvars('TFW_', 'tfwenvtuple') diff --git a/lib/tfw/fsm_base.py b/lib/tfw/fsm_base.py index d6a586c..fc1300f 100644 --- a/lib/tfw/fsm_base.py +++ b/lib/tfw/fsm_base.py @@ -1,11 +1,14 @@ +from typing import List + from transitions import Machine class FSMBase: states, transitions = [], [] - def __init__(self, initial: str = None): + def __init__(self, initial: str = None, accepted_states: List[str] = None): self.message_handlers = [] + self.accepted_states = accepted_states or [self.states[-1]] self.machine = Machine(model=self, states=self.states, transitions=self.transitions, @@ -24,3 +27,6 @@ class FSMBase: def unsubscribe_message_handler(self, msghandler): self.message_handlers.remove(msghandler) + + def is_solved(self): + return self.state in self.accepted_states diff --git a/lib/tfw/networking/controller_connector.py b/lib/tfw/networking/controller_connector.py new file mode 100644 index 0000000..64f83b2 --- /dev/null +++ b/lib/tfw/networking/controller_connector.py @@ -0,0 +1,18 @@ +import zmq +from zmq.eventloop import ioloop +from zmq.eventloop.zmqstream import ZMQStream + +from tfw.config import tfwenv +from tfw.util import ZMQConnectorBase + +ioloop.install() + + +class ControllerConnector(ZMQConnectorBase): + def __init__(self, zmq_context=None): + super(ControllerConnector, self).__init__(zmq_context) + self._zmq_rep_socket = self._zmq_context.socket(zmq.REP) + self._zmq_rep_socket.connect('tcp://localhost:{}'.format(tfwenv.CONTROLLER_PORT)) + self._zmq_rep_stream = ZMQStream(self._zmq_rep_socket) + + self.register_callback = self._zmq_rep_stream.on_recv_stream diff --git a/lib/tfw/networking/serialization.py b/lib/tfw/networking/serialization.py index 7333873..73acc27 100644 --- a/lib/tfw/networking/serialization.py +++ b/lib/tfw/networking/serialization.py @@ -13,9 +13,9 @@ def decode_if_needed(value): return value -def serialize_all(anchor, message): - return [encode_if_needed(a) for a in (anchor, json.dumps(message))] +def serialize_all(key, data): + return [encode_if_needed(frame) for frame in (key, json.dumps(data))] -def deserialize_all(anchor, message): - return decode_if_needed(anchor), json.loads(message) +def deserialize_all(key, data): + return decode_if_needed(key), json.loads(data) diff --git a/lib/tfw/networking/server/tfw_server.py b/lib/tfw/networking/server/tfw_server.py new file mode 100644 index 0000000..fbd5ec3 --- /dev/null +++ b/lib/tfw/networking/server/tfw_server.py @@ -0,0 +1,33 @@ +from tornado.web import Application + +from tfw.networking.controller_connector import ControllerConnector +from tfw.networking.serialization import deserialize_all, serialize_all +from tfw.networking.server.zmq_websocket_handler import FSMManagingSocketHandler + + +class TFWServer: + def __init__(self, fsm_type): + self._fsm = fsm_type() + self.application = Application( + [(r'/ws', FSMManagingSocketHandler, {'fsm': self.fsm})], + autoreload=True + ) + self.controller_connector = ControllerConnector() + self.controller_connector.register_callback(self.zmq_callback) + + @property + def fsm(self): + return self._fsm + + def zmq_callback(self, stream, msg_parts): + key, data = deserialize_all(*msg_parts) + if key == 'test': + stream.send_multipart(serialize_all(key, 'OK')) + if key == 'solution_check': + stream.send_multipart(serialize_all(key, { + 'solved': self.fsm.is_solved(), + 'message': 'solved' if self.fsm.is_solved() else 'not solved' + })) + + def listen(self, port): + self.application.listen(port) diff --git a/lib/tfw/networking/server/zmq_websocket_handler.py b/lib/tfw/networking/server/zmq_websocket_handler.py index 3262f2a..35d5fec 100644 --- a/lib/tfw/networking/server/zmq_websocket_handler.py +++ b/lib/tfw/networking/server/zmq_websocket_handler.py @@ -26,15 +26,9 @@ class ZMQWebSocketHandler(WebSocketHandler): log.debug('Received on WebSocket: {}'.format(message)) self.send_message(*self.make_response(json.loads(message))) - def make_response(self, message): - raise NotImplementedError - def send_message(self, message: dict, anchor: str = None): self._event_handler_connector.send_message(message, anchor) - def on_close(self): - pass - # much secure, very cors, wow def check_origin(self, origin): return True diff --git a/lib/tfw/networking/solvable_connector.py b/lib/tfw/networking/solvable_connector.py new file mode 100644 index 0000000..7a77a57 --- /dev/null +++ b/lib/tfw/networking/solvable_connector.py @@ -0,0 +1,21 @@ +import zmq +from zmq.eventloop import ioloop + +from tfw.config import tfwenv +from tfw.util import ZMQConnectorBase +from tfw.networking.serialization import serialize_all, deserialize_all + +ioloop.install() + + +class SolvableConnector(ZMQConnectorBase): + def __init__(self, zmq_context=None): + super(SolvableConnector, self).__init__(zmq_context) + self._zmq_req_socket = self._zmq_context.socket(zmq.REQ) + self._zmq_req_socket.bind('tcp://*:{}'.format(tfwenv.CONTROLLER_PORT)) + + def send(self, key, message): + self._zmq_req_socket.send_multipart(serialize_all(key, message)) + + def recv(self): + return deserialize_all(*self._zmq_req_socket.recv_multipart()) diff --git a/src/controller/__init__.py b/src/controller/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/controller/app.py b/src/controller/app.py new file mode 100644 index 0000000..c304231 --- /dev/null +++ b/src/controller/app.py @@ -0,0 +1,23 @@ +from tornado.ioloop import IOLoop +from tornado.web import Application + +from tao.config import taoenv +from tfw.config import tfwenv +from handlers import SolutionCheckHandler, TestHandler +from tfw.networking.solvable_connector import SolvableConnector + +from tfw.config.logs import logging +log = logging.getLogger(__name__) + +if __name__ == '__main__': + solvable_connector = SolvableConnector() + routes = [ + (r'/{secret}/?'.format(secret=taoenv.SECRET), SolutionCheckHandler, {'solvable_connector': solvable_connector}), + (r'/{secret}/test/?'.format(secret=taoenv.SECRET), TestHandler, {'solvable_connector': solvable_connector}) + ] + app = Application( + routes + ) + app.listen(tfwenv.CRP_LISTENER_PORT) + log.debug('Controller listening on {}'.format(tfwenv.CRP_LISTENER_PORT)) + IOLoop.instance().start() diff --git a/src/controller/handlers/__init__.py b/src/controller/handlers/__init__.py new file mode 100644 index 0000000..ac7e0ba --- /dev/null +++ b/src/controller/handlers/__init__.py @@ -0,0 +1,2 @@ +from .solution_check_handler import SolutionCheckHandler +from .test_handler import TestHandler diff --git a/src/controller/handlers/solution_check_handler.py b/src/controller/handlers/solution_check_handler.py new file mode 100644 index 0000000..5a79feb --- /dev/null +++ b/src/controller/handlers/solution_check_handler.py @@ -0,0 +1,15 @@ +from tornado.web import RequestHandler + +from tfw.config.logs import logging +log = logging.getLogger(__name__) + +class SolutionCheckHandler(RequestHandler): + def initialize(self, solvable_connector): + self.solvable_connector = solvable_connector + + def get(self): + log.debug('Sending request to solvable') + self.solvable_connector.send('solution_check', {}) + resp_key, resp_data = self.solvable_connector.recv() + log.debug('Received answer from solvable') + self.write(resp_data) diff --git a/src/controller/handlers/test_handler.py b/src/controller/handlers/test_handler.py new file mode 100644 index 0000000..bc4d690 --- /dev/null +++ b/src/controller/handlers/test_handler.py @@ -0,0 +1,11 @@ +from tornado.web import RequestHandler + + +class TestHandler(RequestHandler): + def initialize(self, solvable_connector): + self.solvable_connector = solvable_connector + + def get(self): + self.solvable_connector.send('test', {}) + resp_key, resp_data = self.solvable_connector.recv() + self.write(resp_data) diff --git a/src/app/app.py b/src/demo/app.py similarity index 63% rename from src/app/app.py rename to src/demo/app.py index 3dc6d8f..de73eee 100644 --- a/src/app/app.py +++ b/src/demo/app.py @@ -1,27 +1,18 @@ import sys import tornado import zmq -from tornado.web import Application from tornado.ioloop import IOLoop from sql_injection_fsm import SQLInjectionFSM +from tfw.networking.server.tfw_server import TFWServer from tfw.config import tfwenv from tfw.config.logs import logging log = logging.getLogger(__name__) -from tfw.networking.server.zmq_websocket_handler import FSMManagingSocketHandler if __name__ == '__main__': - fsm = SQLInjectionFSM() - routes = [ - (r'/ws', FSMManagingSocketHandler, {'fsm': fsm}), - ] - application = Application( - routes, - autoreload=True - ) - - application.listen(tfwenv.WEB_PORT) + server = TFWServer(SQLInjectionFSM) + server.listen(tfwenv.WEB_PORT) log.debug('Python version: {}'.format(sys.version[:5])) log.debug('Tornado version: {}'.format(tornado.version)) log.debug('ZeroMQ version: {}'.format(zmq.zmq_version())) diff --git a/src/event_handlers/event_handler_main.py b/src/demo/event_handler_main.py similarity index 65% rename from src/event_handlers/event_handler_main.py rename to src/demo/event_handler_main.py index b349b6b..f7623a9 100644 --- a/src/event_handlers/event_handler_main.py +++ b/src/demo/event_handler_main.py @@ -1,5 +1,5 @@ -from source_code_event_handler import SourceCodeEventHandler -from terminado_event_handler import TerminadoEventHandler +from tfw.components.source_code_event_handler import SourceCodeEventHandler +from tfw.components.terminado_event_handler import TerminadoEventHandler from tornado.ioloop import IOLoop from tfw.config import tfwenv diff --git a/src/event_handlers/source_code_server/login_component.py b/src/demo/source_code_server/login_component.py similarity index 100% rename from src/event_handlers/source_code_server/login_component.py rename to src/demo/source_code_server/login_component.py diff --git a/src/event_handlers/source_code_server/server.py b/src/demo/source_code_server/server.py similarity index 100% rename from src/event_handlers/source_code_server/server.py rename to src/demo/source_code_server/server.py diff --git a/src/event_handlers/source_code_server/users.db b/src/demo/source_code_server/users.db similarity index 100% rename from src/event_handlers/source_code_server/users.db rename to src/demo/source_code_server/users.db diff --git a/src/app/sql_injection_fsm.py b/src/demo/sql_injection_fsm.py similarity index 100% rename from src/app/sql_injection_fsm.py rename to src/demo/sql_injection_fsm.py diff --git a/supervisor/components/controller.conf b/supervisor/components/controller.conf new file mode 100644 index 0000000..1cdb672 --- /dev/null +++ b/supervisor/components/controller.conf @@ -0,0 +1,3 @@ +[program:controller] +directory=%(ENV_TFW_CONTROLLER_DIR)s +command=env python app.py \ No newline at end of file diff --git a/supervisor/components/event_handlers.conf b/supervisor/components/event_handlers.conf index 887c1c6..a28afe5 100644 --- a/supervisor/components/event_handlers.conf +++ b/supervisor/components/event_handlers.conf @@ -1,3 +1,3 @@ [program:event_handler_main] -directory=%(ENV_TFW_EVENT_HANDLERS_DIR)s +directory=%(ENV_TFW_APP_DIR)s command=env python event_handler_main.py diff --git a/supervisor/components/terminado.conf b/supervisor/components/terminado.conf index ccf7595..30c441d 100644 --- a/supervisor/components/terminado.conf +++ b/supervisor/components/terminado.conf @@ -1,4 +1,4 @@ [program:terminado] directory=%(ENV_TFW_TERMINADO_DIR)s -command=env python server.py +command=env python terminado_mini_server.py autostart=false