mrtoth/baseimage-tutorial-framework
1
0
Fork 0
mirror of https://github.com/avatao-content/baseimage-tutorial-framework synced 2024-12-23 02:51:57 +00:00
Code Issues Releases Wiki Activity

Merge branch 'controller'

Browse Source
This commit is contained in:
Kristóf Tóth 2018-02-12 13:44:27 +01:00
commit 4582538060
29 changed files with 217 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
Dockerfile
View File

@ -1,19 +1,19 @@
FROM avatao/ubuntu:16.04
RUN curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - &&\
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - &&\
RUN curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - &&\
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - &&\
echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list &&\
apt-get update &&\
apt-get update &&\
apt-get install -y --no-install-recommends \
nodejs \
yarn \
supervisor \
libzmq5 \
nginx \
gettext-base \
libbz2-dev \
libreadline-dev \
libsqlite3-dev &&\
nodejs \
yarn \
supervisor \
libzmq5 \
nginx \
gettext-base \
libbz2-dev \
libreadline-dev \
libsqlite3-dev &&\
rm -rf /var/lib/apt/lists/*
USER ${AVATAO_USER}
@ -23,14 +23,14 @@ COPY requirements.txt /tmp
ARG PYTHON_VERSION="3.6.4"
ARG PYENV_INSTALLER_URL="https://raw.githubusercontent.com/pyenv/pyenv-installer/78cfd4d/bin/pyenv-installer"
ARG PYENV_INSTALLER_HASH=9509348b828f0564358fff456f7f693dd9ace351dc3f240854d7685ad8a8e1dd
RUN curl -fSL -o pyenv-installer ${PYENV_INSTALLER_URL} &&\
RUN curl -fSL -o pyenv-installer ${PYENV_INSTALLER_URL} &&\
echo "${PYENV_INSTALLER_HASH} *pyenv-installer" | sha256sum -c - &&\
bash pyenv-installer &&\
rm pyenv-installer &&\
echo "source $HOME/.pyenvrc" >> .bashrc &&\
. $HOME/.pyenvrc &&\
pyenv install ${PYTHON_VERSION} &&\
pyenv global ${PYTHON_VERSION} &&\
bash pyenv-installer &&\
rm pyenv-installer &&\
echo "source $HOME/.pyenvrc" >> .bashrc &&\
. $HOME/.pyenvrc &&\
pyenv install ${PYTHON_VERSION} &&\
pyenv global ${PYTHON_VERSION} &&\
pip install -r /tmp/requirements.txt
USER root
@ -39,49 +39,57 @@ COPY src/frontend /data/
RUN yarn install --frozen-lockfile
RUN yarn build --no-progress
ENV TFW_WEB_PORT=4242
ENV TFW_LOGIN_APP_PORT=6666
ENV TFW_TERMINADO_PORT=7878
ENV TFW_SUPERVISOR_HTTP_PORT=9001
ENV TFW_PUBLIC_PORT=8888
ENV TFW_PUBLISHER_PORT=7654
ENV TFW_RECEIVER_PORT=8765
EXPOSE ${TFW_PUBLIC_PORT}
ENV TFW_PUBLIC_PORT=8888 \
TFW_WEB_PORT=4242 \
TFW_LOGIN_APP_PORT=6666 \
TFW_TERMINADO_PORT=7878 \
TFW_SUPERVISOR_HTTP_PORT=9001 \
TFW_CONTROLLER_PORT=7777 \
TFW_CRP_LISTENER_PORT=5555 \
TFW_PUBLISHER_PORT=7654 \
TFW_RECEIVER_PORT=8765
ENV TFW_SUPERVISOR_HTTP_URI="http://localhost:${TFW_SUPERVISOR_HTTP_PORT}"
ENV TFW_EVENT_HANDLERS_DIR="/opt/event_handlers"
ENV TFW_APP_DIR="/srv/app"
ENV TFW_FRONTEND_DIR="/srv/frontend"
ENV TFW_LOGIN_APP_DIR="/tmp/source_code_server"
ENV TFW_TERMINADO_DIR="/tmp/terminado_server"
ENV TFW_WEBIDE_WD="/home/${AVATAO_USER}/workdir"
ENV TFW_TERMINADO_WD=${TFW_WEBIDE_WD}
ENV TFW_LIB_DIR="/usr/local/lib/"
ENV TFW_SUPERVISORD_CONF="/etc/supervisor/supervisord.conf"
ENV TFW_SUPERVISORD_COMPONENTS="/etc/supervisor/conf"
ENV TFW_NGINX_CONF="/etc/nginx/sites-enabled/default"
ENV TFW_NGINX_COMPONENTS="/etc/nginx/components"
ENV PYTHONPATH=${TFW_LIB_DIR}
EXPOSE ${TFW_PUBLIC_PORT} ${TFW_CRP_LISTENER_PORT}
ENV PYTHONPATH="/usr/local/lib/" \
TFW_SUPERVISOR_HTTP_URI="http://localhost:${TFW_SUPERVISOR_HTTP_PORT}" \
TFW_SUPERVISORD_CONF="/etc/supervisor/supervisord.conf" \
TFW_SUPERVISORD_COMPONENTS="/etc/supervisor/conf" \
TFW_NGINX_CONF="/etc/nginx/sites-enabled/default" \
TFW_NGINX_COMPONENTS="/etc/nginx/components" \
TFW_LIB_DIR="/usr/local/lib/" \
TFW_CONTROLLER_DIR="/srv/controller" \
TFW_TERMINADO_DIR="/tmp/terminado_server" \
TFW_FRONTEND_DIR="/srv/frontend"
COPY nginx/nginx.conf ${TFW_NGINX_CONF}
COPY nginx/components/ ${TFW_NGINX_COMPONENTS}
RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\
sed -i 's#pid /run/nginx.pid;#pid /tmp/nginx.pid;#g' /etc/nginx/nginx.conf &&\
for f in "${TFW_NGINX_CONF}" ${TFW_NGINX_COMPONENTS}/*.conf; do \
RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\
sed -i 's#pid /run/nginx.pid;#pid /tmp/nginx.pid;#g' /etc/nginx/nginx.conf &&\
for f in "${TFW_NGINX_CONF}" ${TFW_NGINX_COMPONENTS}/*.conf; do \
envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f ;\
done
COPY lib ${TFW_LIB_DIR}
COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF}
COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS}
COPY src/app ${TFW_APP_DIR}
COPY src/event_handlers ${TFW_EVENT_HANDLERS_DIR}
COPY lib ${TFW_LIB_DIR}
COPY src/controller ${TFW_CONTROLLER_DIR}
COPY lib/tfw/components/terminado_mini_server.py ${TFW_TERMINADO_DIR}/
### TFW internals ^ ### DEMO v ###############################################################
ENV TFW_APP_DIR="/srv/app" \
TFW_LOGIN_APP_DIR="/tmp/source_code_server" \
TFW_WEBIDE_WD="/home/${AVATAO_USER}/workdir" \
TFW_TERMINADO_WD="/home/${AVATAO_USER}/workdir"
RUN mv /data/dist ${TFW_FRONTEND_DIR}
COPY src/event_handlers/source_code_server/server.py ${TFW_LOGIN_APP_DIR}/
COPY src/event_handlers/source_code_server/users.db ${TFW_LOGIN_APP_DIR}/
COPY src/event_handlers/source_code_server/login_component.py ${TFW_WEBIDE_WD}/
COPY src/demo ${TFW_APP_DIR}/
COPY src/demo/source_code_server/server.py ${TFW_LOGIN_APP_DIR}/
COPY src/demo/source_code_server/users.db ${TFW_LOGIN_APP_DIR}/
COPY src/demo/source_code_server/login_component.py ${TFW_WEBIDE_WD}/
RUN chown -R ${AVATAO_USER} ${TFW_WEBIDE_WD} && chmod -R 755 ${TFW_WEBIDE_WD}

9
lib/envvars.py Normal file
View File

@ -0,0 +1,9 @@
from collections import namedtuple
from os import environ
def generate_namedtuple_from_prefixed_envvars(prefix: str, tuple_name: str):
envvars = {envvar.replace(prefix, '', 1): environ.get(envvar)
for envvar in environ.keys()
if envvar.startswith(prefix)}
return namedtuple(tuple_name, envvars)(**envvars)

0
lib/tao/__init__.py Normal file
View File

1
lib/tao/config/__init__.py Normal file
View File

@ -0,0 +1 @@
from .envvars import *

3
lib/tao/config/envvars.py Normal file
View File

@ -0,0 +1,3 @@
from envvars import generate_namedtuple_from_prefixed_envvars
taoenv = generate_namedtuple_from_prefixed_envvars('AVATAO_', 'taoenvtuple')

0
src/event_handlers/source_code_event_handler.py → lib/tfw/components/source_code_event_handler.py
View File

7
src/event_handlers/terminado_event_handler.py → lib/tfw/components/terminado_event_handler.py
View File

@ -1,5 +1,3 @@
from shutil import rmtree, copytree
from tfw.event_handler_base import EventHandlerBase
from tfw.util import SupervisorMixin
from tfw.config import tfwenv
@ -12,11 +10,6 @@ class TerminadoEventHandler(EventHandlerBase, SupervisorMixin):
super().__init__(anchor)
self.working_directory = tfwenv.TERMINADO_DIR
self.process_name = process_name
self.setup_terminado_server()
def setup_terminado_server(self):
rmtree(self.working_directory, ignore_errors=True)
copytree('terminado_server/', self.working_directory)
self.start_process()
def handle_event(self, anchor, data_json):

0
src/event_handlers/terminado_server/server.py → lib/tfw/components/terminado_mini_server.py
View File

9
lib/tfw/config/envvars.py
View File

@ -1,8 +1,3 @@
from os import environ
from collections import namedtuple
from envvars import generate_namedtuple_from_prefixed_envvars
TFW_PREFIX = 'TFW_'
tfwenvvars = {envvar.replace(TFW_PREFIX, '', 1): environ.get(envvar)
for envvar in environ.keys()
if envvar.startswith(TFW_PREFIX)}
tfwenv = namedtuple('tfwenvtuple', tfwenvvars)(**tfwenvvars)
tfwenv = generate_namedtuple_from_prefixed_envvars('TFW_', 'tfwenvtuple')

8
lib/tfw/fsm_base.py
View File

@ -1,11 +1,14 @@
from typing import List
from transitions import Machine
class FSMBase:
states, transitions = [], []
def __init__(self, initial: str = None):
def __init__(self, initial: str = None, accepted_states: List[str] = None):
self.message_handlers = []
self.accepted_states = accepted_states or [self.states[-1]]
self.machine = Machine(model=self,
states=self.states,
transitions=self.transitions,
@ -24,3 +27,6 @@ class FSMBase:
def unsubscribe_message_handler(self, msghandler):
self.message_handlers.remove(msghandler)
def is_solved(self):
return self.state in self.accepted_states

18
lib/tfw/networking/controller_connector.py Normal file
View File

@ -0,0 +1,18 @@
import zmq
from zmq.eventloop import ioloop
from zmq.eventloop.zmqstream import ZMQStream
from tfw.config import tfwenv
from tfw.util import ZMQConnectorBase
ioloop.install()
class ControllerConnector(ZMQConnectorBase):
def __init__(self, zmq_context=None):
super(ControllerConnector, self).__init__(zmq_context)
self._zmq_rep_socket = self._zmq_context.socket(zmq.REP)
self._zmq_rep_socket.connect('tcp://localhost:{}'.format(tfwenv.CONTROLLER_PORT))
self._zmq_rep_stream = ZMQStream(self._zmq_rep_socket)
self.register_callback = self._zmq_rep_stream.on_recv_stream

8
lib/tfw/networking/serialization.py
View File

@ -13,9 +13,9 @@ def decode_if_needed(value):
return value
def serialize_all(anchor, message):
return [encode_if_needed(a) for a in (anchor, json.dumps(message))]
def serialize_all(key, data):
return [encode_if_needed(frame) for frame in (key, json.dumps(data))]
def deserialize_all(anchor, message):
return decode_if_needed(anchor), json.loads(message)
def deserialize_all(key, data):
return decode_if_needed(key), json.loads(data)

33
lib/tfw/networking/server/tfw_server.py Normal file
View File

@ -0,0 +1,33 @@
from tornado.web import Application
from tfw.networking.controller_connector import ControllerConnector
from tfw.networking.serialization import deserialize_all, serialize_all
from tfw.networking.server.zmq_websocket_handler import FSMManagingSocketHandler
class TFWServer:
def __init__(self, fsm_type):
self._fsm = fsm_type()
self.application = Application(
[(r'/ws', FSMManagingSocketHandler, {'fsm': self.fsm})],
autoreload=True
)
self.controller_connector = ControllerConnector()
self.controller_connector.register_callback(self.zmq_callback)
@property
def fsm(self):
return self._fsm
def zmq_callback(self, stream, msg_parts):
key, data = deserialize_all(*msg_parts)
if key == 'test':
stream.send_multipart(serialize_all(key, 'OK'))
if key == 'solution_check':
stream.send_multipart(serialize_all(key, {
'solved': self.fsm.is_solved(),
'message': 'solved' if self.fsm.is_solved() else 'not solved'
}))
def listen(self, port):
self.application.listen(port)

6
lib/tfw/networking/server/zmq_websocket_handler.py
View File

@ -26,15 +26,9 @@ class ZMQWebSocketHandler(WebSocketHandler):
log.debug('Received on WebSocket: {}'.format(message))
self.send_message(*self.make_response(json.loads(message)))
def make_response(self, message):
raise NotImplementedError
def send_message(self, message: dict, anchor: str = None):
self._event_handler_connector.send_message(message, anchor)
def on_close(self):
pass
# much secure, very cors, wow
def check_origin(self, origin):
return True

21
lib/tfw/networking/solvable_connector.py Normal file
View File

@ -0,0 +1,21 @@
import zmq
from zmq.eventloop import ioloop
from tfw.config import tfwenv
from tfw.util import ZMQConnectorBase
from tfw.networking.serialization import serialize_all, deserialize_all
ioloop.install()
class SolvableConnector(ZMQConnectorBase):
def __init__(self, zmq_context=None):
super(SolvableConnector, self).__init__(zmq_context)
self._zmq_req_socket = self._zmq_context.socket(zmq.REQ)
self._zmq_req_socket.bind('tcp://*:{}'.format(tfwenv.CONTROLLER_PORT))
def send(self, key, message):
self._zmq_req_socket.send_multipart(serialize_all(key, message))
def recv(self):
return deserialize_all(*self._zmq_req_socket.recv_multipart())

0
src/controller/__init__.py Normal file
View File

23
src/controller/app.py Normal file
View File

@ -0,0 +1,23 @@
from tornado.ioloop import IOLoop
from tornado.web import Application
from tao.config import taoenv
from tfw.config import tfwenv
from handlers import SolutionCheckHandler, TestHandler
from tfw.networking.solvable_connector import SolvableConnector
from tfw.config.logs import logging
log = logging.getLogger(__name__)
if __name__ == '__main__':
solvable_connector = SolvableConnector()
routes = [
(r'/{secret}/?'.format(secret=taoenv.SECRET), SolutionCheckHandler, {'solvable_connector': solvable_connector}),
(r'/{secret}/test/?'.format(secret=taoenv.SECRET), TestHandler, {'solvable_connector': solvable_connector})
]
app = Application(
routes
)
app.listen(tfwenv.CRP_LISTENER_PORT)
log.debug('Controller listening on {}'.format(tfwenv.CRP_LISTENER_PORT))
IOLoop.instance().start()

2
src/controller/handlers/__init__.py Normal file
View File

@ -0,0 +1,2 @@
from .solution_check_handler import SolutionCheckHandler
from .test_handler import TestHandler

15
src/controller/handlers/solution_check_handler.py Normal file
View File

@ -0,0 +1,15 @@
from tornado.web import RequestHandler
from tfw.config.logs import logging
log = logging.getLogger(__name__)
class SolutionCheckHandler(RequestHandler):
def initialize(self, solvable_connector):
self.solvable_connector = solvable_connector
def get(self):
log.debug('Sending request to solvable')
self.solvable_connector.send('solution_check', {})
resp_key, resp_data = self.solvable_connector.recv()
log.debug('Received answer from solvable')
self.write(resp_data)

11
src/controller/handlers/test_handler.py Normal file
View File

@ -0,0 +1,11 @@
from tornado.web import RequestHandler
class TestHandler(RequestHandler):
def initialize(self, solvable_connector):
self.solvable_connector = solvable_connector
def get(self):
self.solvable_connector.send('test', {})
resp_key, resp_data = self.solvable_connector.recv()
self.write(resp_data)

15
src/app/app.py → src/demo/app.py
View File

@ -1,27 +1,18 @@
import sys
import tornado
import zmq
from tornado.web import Application
from tornado.ioloop import IOLoop
from sql_injection_fsm import SQLInjectionFSM
from tfw.networking.server.tfw_server import TFWServer
from tfw.config import tfwenv
from tfw.config.logs import logging
log = logging.getLogger(__name__)
from tfw.networking.server.zmq_websocket_handler import FSMManagingSocketHandler
if __name__ == '__main__':
fsm = SQLInjectionFSM()
routes = [
(r'/ws', FSMManagingSocketHandler, {'fsm': fsm}),
]
application = Application(
routes,
autoreload=True
)
application.listen(tfwenv.WEB_PORT)
server = TFWServer(SQLInjectionFSM)
server.listen(tfwenv.WEB_PORT)
log.debug('Python version: {}'.format(sys.version[:5]))
log.debug('Tornado version: {}'.format(tornado.version))
log.debug('ZeroMQ version: {}'.format(zmq.zmq_version()))

4
src/event_handlers/event_handler_main.py → src/demo/event_handler_main.py
View File

@ -1,5 +1,5 @@
from source_code_event_handler import SourceCodeEventHandler
from terminado_event_handler import TerminadoEventHandler
from tfw.components.source_code_event_handler import SourceCodeEventHandler
from tfw.components.terminado_event_handler import TerminadoEventHandler
from tornado.ioloop import IOLoop
from tfw.config import tfwenv

0
src/event_handlers/source_code_server/login_component.py → src/demo/source_code_server/login_component.py
View File

0
src/event_handlers/source_code_server/server.py → src/demo/source_code_server/server.py
View File

0
src/event_handlers/source_code_server/users.db → src/demo/source_code_server/users.db
View File

0
src/app/sql_injection_fsm.py → src/demo/sql_injection_fsm.py
View File

3
supervisor/components/controller.conf Normal file
View File

@ -0,0 +1,3 @@
[program:controller]
directory=%(ENV_TFW_CONTROLLER_DIR)s
command=env python app.py

2
supervisor/components/event_handlers.conf
View File

@ -1,3 +1,3 @@
[program:event_handler_main]
directory=%(ENV_TFW_EVENT_HANDLERS_DIR)s
directory=%(ENV_TFW_APP_DIR)s
command=env python event_handler_main.py

2
supervisor/components/terminado.conf
View File

@ -1,4 +1,4 @@
[program:terminado]
directory=%(ENV_TFW_TERMINADO_DIR)s
command=env python server.py
command=env python terminado_mini_server.py
autostart=false