mirror of
https://github.com/avatao-content/baseimage-tutorial-framework
synced 2024-11-05 16:41:21 +00:00
Merge branch 'controller'
This commit is contained in:
commit
4582538060
108
Dockerfile
108
Dockerfile
@ -1,19 +1,19 @@
|
||||
FROM avatao/ubuntu:16.04
|
||||
|
||||
RUN curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - &&\
|
||||
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - &&\
|
||||
RUN curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - &&\
|
||||
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - &&\
|
||||
echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list &&\
|
||||
apt-get update &&\
|
||||
apt-get update &&\
|
||||
apt-get install -y --no-install-recommends \
|
||||
nodejs \
|
||||
yarn \
|
||||
supervisor \
|
||||
libzmq5 \
|
||||
nginx \
|
||||
gettext-base \
|
||||
libbz2-dev \
|
||||
libreadline-dev \
|
||||
libsqlite3-dev &&\
|
||||
nodejs \
|
||||
yarn \
|
||||
supervisor \
|
||||
libzmq5 \
|
||||
nginx \
|
||||
gettext-base \
|
||||
libbz2-dev \
|
||||
libreadline-dev \
|
||||
libsqlite3-dev &&\
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
USER ${AVATAO_USER}
|
||||
@ -23,14 +23,14 @@ COPY requirements.txt /tmp
|
||||
ARG PYTHON_VERSION="3.6.4"
|
||||
ARG PYENV_INSTALLER_URL="https://raw.githubusercontent.com/pyenv/pyenv-installer/78cfd4d/bin/pyenv-installer"
|
||||
ARG PYENV_INSTALLER_HASH=9509348b828f0564358fff456f7f693dd9ace351dc3f240854d7685ad8a8e1dd
|
||||
RUN curl -fSL -o pyenv-installer ${PYENV_INSTALLER_URL} &&\
|
||||
RUN curl -fSL -o pyenv-installer ${PYENV_INSTALLER_URL} &&\
|
||||
echo "${PYENV_INSTALLER_HASH} *pyenv-installer" | sha256sum -c - &&\
|
||||
bash pyenv-installer &&\
|
||||
rm pyenv-installer &&\
|
||||
echo "source $HOME/.pyenvrc" >> .bashrc &&\
|
||||
. $HOME/.pyenvrc &&\
|
||||
pyenv install ${PYTHON_VERSION} &&\
|
||||
pyenv global ${PYTHON_VERSION} &&\
|
||||
bash pyenv-installer &&\
|
||||
rm pyenv-installer &&\
|
||||
echo "source $HOME/.pyenvrc" >> .bashrc &&\
|
||||
. $HOME/.pyenvrc &&\
|
||||
pyenv install ${PYTHON_VERSION} &&\
|
||||
pyenv global ${PYTHON_VERSION} &&\
|
||||
pip install -r /tmp/requirements.txt
|
||||
|
||||
USER root
|
||||
@ -39,49 +39,57 @@ COPY src/frontend /data/
|
||||
RUN yarn install --frozen-lockfile
|
||||
RUN yarn build --no-progress
|
||||
|
||||
ENV TFW_WEB_PORT=4242
|
||||
ENV TFW_LOGIN_APP_PORT=6666
|
||||
ENV TFW_TERMINADO_PORT=7878
|
||||
ENV TFW_SUPERVISOR_HTTP_PORT=9001
|
||||
ENV TFW_PUBLIC_PORT=8888
|
||||
ENV TFW_PUBLISHER_PORT=7654
|
||||
ENV TFW_RECEIVER_PORT=8765
|
||||
EXPOSE ${TFW_PUBLIC_PORT}
|
||||
ENV TFW_PUBLIC_PORT=8888 \
|
||||
TFW_WEB_PORT=4242 \
|
||||
TFW_LOGIN_APP_PORT=6666 \
|
||||
TFW_TERMINADO_PORT=7878 \
|
||||
TFW_SUPERVISOR_HTTP_PORT=9001 \
|
||||
TFW_CONTROLLER_PORT=7777 \
|
||||
TFW_CRP_LISTENER_PORT=5555 \
|
||||
TFW_PUBLISHER_PORT=7654 \
|
||||
TFW_RECEIVER_PORT=8765
|
||||
|
||||
ENV TFW_SUPERVISOR_HTTP_URI="http://localhost:${TFW_SUPERVISOR_HTTP_PORT}"
|
||||
ENV TFW_EVENT_HANDLERS_DIR="/opt/event_handlers"
|
||||
ENV TFW_APP_DIR="/srv/app"
|
||||
ENV TFW_FRONTEND_DIR="/srv/frontend"
|
||||
ENV TFW_LOGIN_APP_DIR="/tmp/source_code_server"
|
||||
ENV TFW_TERMINADO_DIR="/tmp/terminado_server"
|
||||
ENV TFW_WEBIDE_WD="/home/${AVATAO_USER}/workdir"
|
||||
ENV TFW_TERMINADO_WD=${TFW_WEBIDE_WD}
|
||||
ENV TFW_LIB_DIR="/usr/local/lib/"
|
||||
ENV TFW_SUPERVISORD_CONF="/etc/supervisor/supervisord.conf"
|
||||
ENV TFW_SUPERVISORD_COMPONENTS="/etc/supervisor/conf"
|
||||
ENV TFW_NGINX_CONF="/etc/nginx/sites-enabled/default"
|
||||
ENV TFW_NGINX_COMPONENTS="/etc/nginx/components"
|
||||
ENV PYTHONPATH=${TFW_LIB_DIR}
|
||||
EXPOSE ${TFW_PUBLIC_PORT} ${TFW_CRP_LISTENER_PORT}
|
||||
|
||||
ENV PYTHONPATH="/usr/local/lib/" \
|
||||
TFW_SUPERVISOR_HTTP_URI="http://localhost:${TFW_SUPERVISOR_HTTP_PORT}" \
|
||||
TFW_SUPERVISORD_CONF="/etc/supervisor/supervisord.conf" \
|
||||
TFW_SUPERVISORD_COMPONENTS="/etc/supervisor/conf" \
|
||||
TFW_NGINX_CONF="/etc/nginx/sites-enabled/default" \
|
||||
TFW_NGINX_COMPONENTS="/etc/nginx/components" \
|
||||
TFW_LIB_DIR="/usr/local/lib/" \
|
||||
TFW_CONTROLLER_DIR="/srv/controller" \
|
||||
TFW_TERMINADO_DIR="/tmp/terminado_server" \
|
||||
TFW_FRONTEND_DIR="/srv/frontend"
|
||||
|
||||
COPY nginx/nginx.conf ${TFW_NGINX_CONF}
|
||||
COPY nginx/components/ ${TFW_NGINX_COMPONENTS}
|
||||
RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\
|
||||
sed -i 's#pid /run/nginx.pid;#pid /tmp/nginx.pid;#g' /etc/nginx/nginx.conf &&\
|
||||
for f in "${TFW_NGINX_CONF}" ${TFW_NGINX_COMPONENTS}/*.conf; do \
|
||||
RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\
|
||||
sed -i 's#pid /run/nginx.pid;#pid /tmp/nginx.pid;#g' /etc/nginx/nginx.conf &&\
|
||||
for f in "${TFW_NGINX_CONF}" ${TFW_NGINX_COMPONENTS}/*.conf; do \
|
||||
envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f ;\
|
||||
done
|
||||
|
||||
COPY lib ${TFW_LIB_DIR}
|
||||
COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF}
|
||||
COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS}
|
||||
COPY src/app ${TFW_APP_DIR}
|
||||
COPY src/event_handlers ${TFW_EVENT_HANDLERS_DIR}
|
||||
COPY lib ${TFW_LIB_DIR}
|
||||
COPY src/controller ${TFW_CONTROLLER_DIR}
|
||||
COPY lib/tfw/components/terminado_mini_server.py ${TFW_TERMINADO_DIR}/
|
||||
|
||||
### TFW internals ^ ### DEMO v ###############################################################
|
||||
|
||||
ENV TFW_APP_DIR="/srv/app" \
|
||||
TFW_LOGIN_APP_DIR="/tmp/source_code_server" \
|
||||
TFW_WEBIDE_WD="/home/${AVATAO_USER}/workdir" \
|
||||
TFW_TERMINADO_WD="/home/${AVATAO_USER}/workdir"
|
||||
|
||||
RUN mv /data/dist ${TFW_FRONTEND_DIR}
|
||||
|
||||
COPY src/event_handlers/source_code_server/server.py ${TFW_LOGIN_APP_DIR}/
|
||||
COPY src/event_handlers/source_code_server/users.db ${TFW_LOGIN_APP_DIR}/
|
||||
COPY src/event_handlers/source_code_server/login_component.py ${TFW_WEBIDE_WD}/
|
||||
COPY src/demo ${TFW_APP_DIR}/
|
||||
|
||||
COPY src/demo/source_code_server/server.py ${TFW_LOGIN_APP_DIR}/
|
||||
COPY src/demo/source_code_server/users.db ${TFW_LOGIN_APP_DIR}/
|
||||
COPY src/demo/source_code_server/login_component.py ${TFW_WEBIDE_WD}/
|
||||
|
||||
RUN chown -R ${AVATAO_USER} ${TFW_WEBIDE_WD} && chmod -R 755 ${TFW_WEBIDE_WD}
|
||||
|
||||
|
9
lib/envvars.py
Normal file
9
lib/envvars.py
Normal file
@ -0,0 +1,9 @@
|
||||
from collections import namedtuple
|
||||
from os import environ
|
||||
|
||||
|
||||
def generate_namedtuple_from_prefixed_envvars(prefix: str, tuple_name: str):
|
||||
envvars = {envvar.replace(prefix, '', 1): environ.get(envvar)
|
||||
for envvar in environ.keys()
|
||||
if envvar.startswith(prefix)}
|
||||
return namedtuple(tuple_name, envvars)(**envvars)
|
0
lib/tao/__init__.py
Normal file
0
lib/tao/__init__.py
Normal file
1
lib/tao/config/__init__.py
Normal file
1
lib/tao/config/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
from .envvars import *
|
3
lib/tao/config/envvars.py
Normal file
3
lib/tao/config/envvars.py
Normal file
@ -0,0 +1,3 @@
|
||||
from envvars import generate_namedtuple_from_prefixed_envvars
|
||||
|
||||
taoenv = generate_namedtuple_from_prefixed_envvars('AVATAO_', 'taoenvtuple')
|
@ -1,5 +1,3 @@
|
||||
from shutil import rmtree, copytree
|
||||
|
||||
from tfw.event_handler_base import EventHandlerBase
|
||||
from tfw.util import SupervisorMixin
|
||||
from tfw.config import tfwenv
|
||||
@ -12,11 +10,6 @@ class TerminadoEventHandler(EventHandlerBase, SupervisorMixin):
|
||||
super().__init__(anchor)
|
||||
self.working_directory = tfwenv.TERMINADO_DIR
|
||||
self.process_name = process_name
|
||||
self.setup_terminado_server()
|
||||
|
||||
def setup_terminado_server(self):
|
||||
rmtree(self.working_directory, ignore_errors=True)
|
||||
copytree('terminado_server/', self.working_directory)
|
||||
self.start_process()
|
||||
|
||||
def handle_event(self, anchor, data_json):
|
@ -1,8 +1,3 @@
|
||||
from os import environ
|
||||
from collections import namedtuple
|
||||
from envvars import generate_namedtuple_from_prefixed_envvars
|
||||
|
||||
TFW_PREFIX = 'TFW_'
|
||||
tfwenvvars = {envvar.replace(TFW_PREFIX, '', 1): environ.get(envvar)
|
||||
for envvar in environ.keys()
|
||||
if envvar.startswith(TFW_PREFIX)}
|
||||
tfwenv = namedtuple('tfwenvtuple', tfwenvvars)(**tfwenvvars)
|
||||
tfwenv = generate_namedtuple_from_prefixed_envvars('TFW_', 'tfwenvtuple')
|
||||
|
@ -1,11 +1,14 @@
|
||||
from typing import List
|
||||
|
||||
from transitions import Machine
|
||||
|
||||
|
||||
class FSMBase:
|
||||
states, transitions = [], []
|
||||
|
||||
def __init__(self, initial: str = None):
|
||||
def __init__(self, initial: str = None, accepted_states: List[str] = None):
|
||||
self.message_handlers = []
|
||||
self.accepted_states = accepted_states or [self.states[-1]]
|
||||
self.machine = Machine(model=self,
|
||||
states=self.states,
|
||||
transitions=self.transitions,
|
||||
@ -24,3 +27,6 @@ class FSMBase:
|
||||
|
||||
def unsubscribe_message_handler(self, msghandler):
|
||||
self.message_handlers.remove(msghandler)
|
||||
|
||||
def is_solved(self):
|
||||
return self.state in self.accepted_states
|
||||
|
18
lib/tfw/networking/controller_connector.py
Normal file
18
lib/tfw/networking/controller_connector.py
Normal file
@ -0,0 +1,18 @@
|
||||
import zmq
|
||||
from zmq.eventloop import ioloop
|
||||
from zmq.eventloop.zmqstream import ZMQStream
|
||||
|
||||
from tfw.config import tfwenv
|
||||
from tfw.util import ZMQConnectorBase
|
||||
|
||||
ioloop.install()
|
||||
|
||||
|
||||
class ControllerConnector(ZMQConnectorBase):
|
||||
def __init__(self, zmq_context=None):
|
||||
super(ControllerConnector, self).__init__(zmq_context)
|
||||
self._zmq_rep_socket = self._zmq_context.socket(zmq.REP)
|
||||
self._zmq_rep_socket.connect('tcp://localhost:{}'.format(tfwenv.CONTROLLER_PORT))
|
||||
self._zmq_rep_stream = ZMQStream(self._zmq_rep_socket)
|
||||
|
||||
self.register_callback = self._zmq_rep_stream.on_recv_stream
|
@ -13,9 +13,9 @@ def decode_if_needed(value):
|
||||
return value
|
||||
|
||||
|
||||
def serialize_all(anchor, message):
|
||||
return [encode_if_needed(a) for a in (anchor, json.dumps(message))]
|
||||
def serialize_all(key, data):
|
||||
return [encode_if_needed(frame) for frame in (key, json.dumps(data))]
|
||||
|
||||
|
||||
def deserialize_all(anchor, message):
|
||||
return decode_if_needed(anchor), json.loads(message)
|
||||
def deserialize_all(key, data):
|
||||
return decode_if_needed(key), json.loads(data)
|
||||
|
33
lib/tfw/networking/server/tfw_server.py
Normal file
33
lib/tfw/networking/server/tfw_server.py
Normal file
@ -0,0 +1,33 @@
|
||||
from tornado.web import Application
|
||||
|
||||
from tfw.networking.controller_connector import ControllerConnector
|
||||
from tfw.networking.serialization import deserialize_all, serialize_all
|
||||
from tfw.networking.server.zmq_websocket_handler import FSMManagingSocketHandler
|
||||
|
||||
|
||||
class TFWServer:
|
||||
def __init__(self, fsm_type):
|
||||
self._fsm = fsm_type()
|
||||
self.application = Application(
|
||||
[(r'/ws', FSMManagingSocketHandler, {'fsm': self.fsm})],
|
||||
autoreload=True
|
||||
)
|
||||
self.controller_connector = ControllerConnector()
|
||||
self.controller_connector.register_callback(self.zmq_callback)
|
||||
|
||||
@property
|
||||
def fsm(self):
|
||||
return self._fsm
|
||||
|
||||
def zmq_callback(self, stream, msg_parts):
|
||||
key, data = deserialize_all(*msg_parts)
|
||||
if key == 'test':
|
||||
stream.send_multipart(serialize_all(key, 'OK'))
|
||||
if key == 'solution_check':
|
||||
stream.send_multipart(serialize_all(key, {
|
||||
'solved': self.fsm.is_solved(),
|
||||
'message': 'solved' if self.fsm.is_solved() else 'not solved'
|
||||
}))
|
||||
|
||||
def listen(self, port):
|
||||
self.application.listen(port)
|
@ -26,15 +26,9 @@ class ZMQWebSocketHandler(WebSocketHandler):
|
||||
log.debug('Received on WebSocket: {}'.format(message))
|
||||
self.send_message(*self.make_response(json.loads(message)))
|
||||
|
||||
def make_response(self, message):
|
||||
raise NotImplementedError
|
||||
|
||||
def send_message(self, message: dict, anchor: str = None):
|
||||
self._event_handler_connector.send_message(message, anchor)
|
||||
|
||||
def on_close(self):
|
||||
pass
|
||||
|
||||
# much secure, very cors, wow
|
||||
def check_origin(self, origin):
|
||||
return True
|
||||
|
21
lib/tfw/networking/solvable_connector.py
Normal file
21
lib/tfw/networking/solvable_connector.py
Normal file
@ -0,0 +1,21 @@
|
||||
import zmq
|
||||
from zmq.eventloop import ioloop
|
||||
|
||||
from tfw.config import tfwenv
|
||||
from tfw.util import ZMQConnectorBase
|
||||
from tfw.networking.serialization import serialize_all, deserialize_all
|
||||
|
||||
ioloop.install()
|
||||
|
||||
|
||||
class SolvableConnector(ZMQConnectorBase):
|
||||
def __init__(self, zmq_context=None):
|
||||
super(SolvableConnector, self).__init__(zmq_context)
|
||||
self._zmq_req_socket = self._zmq_context.socket(zmq.REQ)
|
||||
self._zmq_req_socket.bind('tcp://*:{}'.format(tfwenv.CONTROLLER_PORT))
|
||||
|
||||
def send(self, key, message):
|
||||
self._zmq_req_socket.send_multipart(serialize_all(key, message))
|
||||
|
||||
def recv(self):
|
||||
return deserialize_all(*self._zmq_req_socket.recv_multipart())
|
0
src/controller/__init__.py
Normal file
0
src/controller/__init__.py
Normal file
23
src/controller/app.py
Normal file
23
src/controller/app.py
Normal file
@ -0,0 +1,23 @@
|
||||
from tornado.ioloop import IOLoop
|
||||
from tornado.web import Application
|
||||
|
||||
from tao.config import taoenv
|
||||
from tfw.config import tfwenv
|
||||
from handlers import SolutionCheckHandler, TestHandler
|
||||
from tfw.networking.solvable_connector import SolvableConnector
|
||||
|
||||
from tfw.config.logs import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
if __name__ == '__main__':
|
||||
solvable_connector = SolvableConnector()
|
||||
routes = [
|
||||
(r'/{secret}/?'.format(secret=taoenv.SECRET), SolutionCheckHandler, {'solvable_connector': solvable_connector}),
|
||||
(r'/{secret}/test/?'.format(secret=taoenv.SECRET), TestHandler, {'solvable_connector': solvable_connector})
|
||||
]
|
||||
app = Application(
|
||||
routes
|
||||
)
|
||||
app.listen(tfwenv.CRP_LISTENER_PORT)
|
||||
log.debug('Controller listening on {}'.format(tfwenv.CRP_LISTENER_PORT))
|
||||
IOLoop.instance().start()
|
2
src/controller/handlers/__init__.py
Normal file
2
src/controller/handlers/__init__.py
Normal file
@ -0,0 +1,2 @@
|
||||
from .solution_check_handler import SolutionCheckHandler
|
||||
from .test_handler import TestHandler
|
15
src/controller/handlers/solution_check_handler.py
Normal file
15
src/controller/handlers/solution_check_handler.py
Normal file
@ -0,0 +1,15 @@
|
||||
from tornado.web import RequestHandler
|
||||
|
||||
from tfw.config.logs import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
class SolutionCheckHandler(RequestHandler):
|
||||
def initialize(self, solvable_connector):
|
||||
self.solvable_connector = solvable_connector
|
||||
|
||||
def get(self):
|
||||
log.debug('Sending request to solvable')
|
||||
self.solvable_connector.send('solution_check', {})
|
||||
resp_key, resp_data = self.solvable_connector.recv()
|
||||
log.debug('Received answer from solvable')
|
||||
self.write(resp_data)
|
11
src/controller/handlers/test_handler.py
Normal file
11
src/controller/handlers/test_handler.py
Normal file
@ -0,0 +1,11 @@
|
||||
from tornado.web import RequestHandler
|
||||
|
||||
|
||||
class TestHandler(RequestHandler):
|
||||
def initialize(self, solvable_connector):
|
||||
self.solvable_connector = solvable_connector
|
||||
|
||||
def get(self):
|
||||
self.solvable_connector.send('test', {})
|
||||
resp_key, resp_data = self.solvable_connector.recv()
|
||||
self.write(resp_data)
|
@ -1,27 +1,18 @@
|
||||
import sys
|
||||
import tornado
|
||||
import zmq
|
||||
from tornado.web import Application
|
||||
from tornado.ioloop import IOLoop
|
||||
|
||||
from sql_injection_fsm import SQLInjectionFSM
|
||||
from tfw.networking.server.tfw_server import TFWServer
|
||||
from tfw.config import tfwenv
|
||||
from tfw.config.logs import logging
|
||||
log = logging.getLogger(__name__)
|
||||
from tfw.networking.server.zmq_websocket_handler import FSMManagingSocketHandler
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
fsm = SQLInjectionFSM()
|
||||
routes = [
|
||||
(r'/ws', FSMManagingSocketHandler, {'fsm': fsm}),
|
||||
]
|
||||
application = Application(
|
||||
routes,
|
||||
autoreload=True
|
||||
)
|
||||
|
||||
application.listen(tfwenv.WEB_PORT)
|
||||
server = TFWServer(SQLInjectionFSM)
|
||||
server.listen(tfwenv.WEB_PORT)
|
||||
log.debug('Python version: {}'.format(sys.version[:5]))
|
||||
log.debug('Tornado version: {}'.format(tornado.version))
|
||||
log.debug('ZeroMQ version: {}'.format(zmq.zmq_version()))
|
@ -1,5 +1,5 @@
|
||||
from source_code_event_handler import SourceCodeEventHandler
|
||||
from terminado_event_handler import TerminadoEventHandler
|
||||
from tfw.components.source_code_event_handler import SourceCodeEventHandler
|
||||
from tfw.components.terminado_event_handler import TerminadoEventHandler
|
||||
from tornado.ioloop import IOLoop
|
||||
|
||||
from tfw.config import tfwenv
|
3
supervisor/components/controller.conf
Normal file
3
supervisor/components/controller.conf
Normal file
@ -0,0 +1,3 @@
|
||||
[program:controller]
|
||||
directory=%(ENV_TFW_CONTROLLER_DIR)s
|
||||
command=env python app.py
|
@ -1,3 +1,3 @@
|
||||
[program:event_handler_main]
|
||||
directory=%(ENV_TFW_EVENT_HANDLERS_DIR)s
|
||||
directory=%(ENV_TFW_APP_DIR)s
|
||||
command=env python event_handler_main.py
|
||||
|
@ -1,4 +1,4 @@
|
||||
[program:terminado]
|
||||
directory=%(ENV_TFW_TERMINADO_DIR)s
|
||||
command=env python server.py
|
||||
command=env python terminado_mini_server.py
|
||||
autostart=false
|
||||
|
Loading…
Reference in New Issue
Block a user