Merge pull request #24 from avatao-content/baseimage_final

Refactor project into TFW baseimage
This commit is contained in:
Bokros Bálint 2018-03-23 15:44:24 +01:00 committed by GitHub
commit 4bf3c68f04
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 25 additions and 315 deletions

View File

@ -34,21 +34,15 @@ RUN curl -fSL -o pyenv-installer ${PYENV_INSTALLER_URL} &&\
pip install -r /tmp/requirements.txt pip install -r /tmp/requirements.txt
USER root USER root
COPY src/frontend /data/
RUN cd /data && yarn install --frozen-lockfile
RUN cd /data && yarn build --no-progress
ENV TFW_PUBLIC_PORT=8888 \ ENV TFW_PUBLIC_PORT=8888 \
TFW_WEB_PORT=4242 \ TFW_WEB_PORT=4242 \
TFW_LOGIN_APP_PORT=6666 \ TFW_LOGIN_APP_PORT=6666 \
TFW_TERMINADO_PORT=7878 \ TFW_TERMINADO_PORT=7878 \
TFW_SUPERVISOR_HTTP_PORT=9001 \ TFW_SUPERVISOR_HTTP_PORT=9001 \
TFW_CONTROLLER_PORT=7777 \
TFW_CRP_LISTENER_PORT=5555 \
TFW_PUBLISHER_PORT=7654 \ TFW_PUBLISHER_PORT=7654 \
TFW_RECEIVER_PORT=8765 TFW_RECEIVER_PORT=8765
EXPOSE ${TFW_PUBLIC_PORT} ${TFW_CRP_LISTENER_PORT} EXPOSE ${TFW_PUBLIC_PORT}
ENV PYTHONPATH="/usr/local/lib/" \ ENV PYTHONPATH="/usr/local/lib/" \
TFW_SUPERVISOR_HTTP_URI="http://localhost:${TFW_SUPERVISOR_HTTP_PORT}" \ TFW_SUPERVISOR_HTTP_URI="http://localhost:${TFW_SUPERVISOR_HTTP_PORT}" \
@ -58,7 +52,6 @@ ENV PYTHONPATH="/usr/local/lib/" \
TFW_NGINX_DEFAULT="/etc/nginx/sites-enabled/default" \ TFW_NGINX_DEFAULT="/etc/nginx/sites-enabled/default" \
TFW_NGINX_COMPONENTS="/etc/nginx/components" \ TFW_NGINX_COMPONENTS="/etc/nginx/components" \
TFW_LIB_DIR="/usr/local/lib/" \ TFW_LIB_DIR="/usr/local/lib/" \
TFW_CONTROLLER_DIR="/srv/controller" \
TFW_TERMINADO_DIR="/tmp/terminado_server" \ TFW_TERMINADO_DIR="/tmp/terminado_server" \
TFW_FRONTEND_DIR="/srv/frontend" \ TFW_FRONTEND_DIR="/srv/frontend" \
TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history" \ TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history" \
@ -73,38 +66,25 @@ RUN echo "shopt -s cmdhist\n" \
'PROMPT_COMMAND="history -a"\n' \ 'PROMPT_COMMAND="history -a"\n' \
>> /home/${AVATAO_USER}/.bashrc >> /home/${AVATAO_USER}/.bashrc
COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF}
COPY nginx/nginx.conf ${TFW_NGINX_CONF} COPY nginx/nginx.conf ${TFW_NGINX_CONF}
COPY nginx/default.conf ${TFW_NGINX_DEFAULT} COPY nginx/default.conf ${TFW_NGINX_DEFAULT}
COPY nginx/components/ ${TFW_NGINX_COMPONENTS} COPY lib ${TFW_LIB_DIR}
RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\
ONBUILD ARG BUILD_CONTEXT="."
ONBUILD ARG NOFRONTEND=""
ONBUILD COPY ${BUILD_CONTEXT}/nginx/components/ ${TFW_NGINX_COMPONENTS}
ONBUILD COPY ${BUILD_CONTEXT}/supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS}
ONBUILD RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\
for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do \ for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do \
envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f~ && mv $f~ $f ;\ envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f~ && mv $f~ $f ;\
done done
COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF} ONBUILD COPY ${BUILD_CONTEXT}/frontend /data/
COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS} ONBUILD RUN if [ -z "${NOFRONTEND}" ]; then cd /data && yarn install --frozen-lockfile; fi
COPY lib ${TFW_LIB_DIR} ONBUILD RUN if [ -z "${NOFRONTEND}" ]; then cd /data && yarn build --no-progress; fi
COPY src/controller ${TFW_CONTROLLER_DIR} ONBUILD RUN if [ -z "${NOFRONTEND}" ]; then mv /data/dist ${TFW_FRONTEND_DIR} && rm -rf /data; fi
RUN mv /data/dist ${TFW_FRONTEND_DIR} && rm -rf /data
### TFW internals ^ ### DEMO v ###############################################################
ENV TFW_APP_DIR="/srv/app" \
TFW_LOGIN_APP_DIR="/tmp/source_code_server" \
TFW_WEBIDE_WD="/home/${AVATAO_USER}/workdir" \
TFW_TERMINADO_WD="/home/${AVATAO_USER}/workdir"
COPY src/demo ${TFW_APP_DIR}/
COPY src/demo/source_code_server/server.py ${TFW_LOGIN_APP_DIR}/
COPY src/demo/source_code_server/users.db ${TFW_LOGIN_APP_DIR}/
COPY src/demo/source_code_server/login_component.py ${TFW_WEBIDE_WD}/
RUN chown -R ${AVATAO_USER} ${TFW_WEBIDE_WD} &&\
chmod -R 755 ${TFW_WEBIDE_WD}
USER ${AVATAO_USER}
VOLUME ["/etc/nginx", "/home/${AVATAO_USER}", "/var/lib/nginx", "/var/log/nginx"]
WORKDIR /home/${AVATAO_USER}
CMD . "$HOME/.pyenvrc" && exec supervisord --nodaemon CMD . "$HOME/.pyenvrc" && exec supervisord --nodaemon

View File

@ -5,17 +5,12 @@ Tutorial Framework
Execute the `tfw_magic_start.sh` script and it will automagically run a backend instance in Docker and serve the Angular frontend locally. Execute the `tfw_magic_start.sh` script and it will automagically run a backend instance in Docker and serve the Angular frontend locally.
This requires the `tutorial-framework-wip` and `tutorial-framework-ng` repositories to have a common parent folder. This requires the `test-tutorial-framework` (Docker image `FROM baseimage-tutorial-framework`) and `frontend-tutorial-framework` repositories to have a common parent folder.
## Building and running with Docker ## Building with Docker
Simply issue `docker build -t tfw .` in the project root. The first build could take a while as it's compiling a fresh Simply issue `docker build -t baseimage-tutorial-framework .` in the project root.
Python package from source.
Run with `docker run --rm -p 8888:8888 -e AVATAO_SECRET=secret tfw`. ## Creating child images
Running locally is possible with lots of pain involved, so this is no longer officially supported. Documentation in progress...
## Frontend
Place an `index.html` and related static files in `src/frontend/dist`, the web server will serve them.

View File

@ -1,14 +0,0 @@
from tfw.networking.serialization import deserialize_all
from tfw.networking.solvable_connector import SolvableConnector
class AsyncSolvableConnector(SolvableConnector):
def __init__(self, async_zmq_context=None):
if async_zmq_context is None:
from zmq.eventloop.future import Context
async_zmq_context = Context.instance()
super(AsyncSolvableConnector, self).__init__(async_zmq_context)
async def recv(self):
response = await self._zmq_req_socket.recv_multipart()
return deserialize_all(*response)

View File

@ -1,7 +1,6 @@
from tornado.web import Application from tornado.web import Application
from collections import defaultdict from collections import defaultdict
from tfw.networking.server.controller_responder import ControllerResponder
from tfw.networking.server.zmq_websocket_handler import ZMQWebSocketProxy from tfw.networking.server.zmq_websocket_handler import ZMQWebSocketProxy
from tfw.networking.event_handlers.server_connector import ServerUplinkConnector from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
from tfw.message_sender import MessageSender from tfw.message_sender import MessageSender
@ -20,7 +19,7 @@ class TFWServer:
[(r'/ws', ZMQWebSocketProxy, {'make_response': self.make_response, [(r'/ws', ZMQWebSocketProxy, {'make_response': self.make_response,
'proxy_filter': self.proxy_filter})] 'proxy_filter': self.proxy_filter})]
) )
self.controller_responder = ControllerResponder(self.fsm) #self.controller_responder = ControllerResponder(self.fsm) TODO: add this once controller stuff is resolved
@property @property
def fsm(self): def fsm(self):

View File

@ -1,18 +0,0 @@
import zmq
from tfw.config import tfwenv
from tfw.networking.zmq_connector_base import ZMQConnectorBase
from tfw.networking.serialization import serialize_all, deserialize_all
class SolvableConnector(ZMQConnectorBase):
def __init__(self, zmq_context=None):
super(SolvableConnector, self).__init__(zmq_context)
self._zmq_req_socket = self._zmq_context.socket(zmq.REQ)
self._zmq_req_socket.bind('tcp://*:{}'.format(tfwenv.CONTROLLER_PORT))
def send(self, key, message):
self._zmq_req_socket.send_multipart(serialize_all(key, message))
def recv(self):
return deserialize_all(*self._zmq_req_socket.recv_multipart())

View File

@ -1,38 +0,0 @@
import secrets
from tornado.ioloop import IOLoop
from tornado.web import Application
from tao.config import taoenv
from tfw.config import tfwenv
from handlers import SolutionCheckHandler, TestHandler
from tfw.networking.async_solvable_connector import AsyncSolvableConnector
from tfw.config.logs import logging
log = logging.getLogger(__name__)
async def setup_token(solvable_connector, token):
solvable_connector.send('token', token)
key, data = await solvable_connector.recv()
if secrets.compare_digest(token, key):
log.debug('Token setup is done.')
else:
log.error('Failed to setup token between controller and solvable.') # TODO: signaling to Avatao platform?
if __name__ == '__main__':
kwargs = {
'solvable_connector': AsyncSolvableConnector(),
'token': secrets.token_hex(32)
}
IOLoop.instance().add_callback(setup_token, **kwargs)
routes = [
(r'/{secret}/?'.format(secret=taoenv.SECRET), SolutionCheckHandler, kwargs),
(r'/{secret}/test/?'.format(secret=taoenv.SECRET), TestHandler, kwargs)
]
app = Application(
routes
)
app.listen(tfwenv.CRP_LISTENER_PORT)
log.debug('Controller listening on {}'.format(tfwenv.CRP_LISTENER_PORT))
IOLoop.instance().start()

View File

@ -1,2 +0,0 @@
from .solution_check_handler import SolutionCheckHandler
from .test_handler import TestHandler

View File

@ -1,19 +0,0 @@
import secrets
from tornado.web import RequestHandler, HTTPError
from tfw.config.logs import logging
log = logging.getLogger(__name__)
class SolutionCheckHandler(RequestHandler):
def initialize(self, solvable_connector, token):
self.solvable_connector = solvable_connector
self.token = token
async def get(self):
log.debug('Sending request to solvable')
self.solvable_connector.send('solution_check', {})
resp_token, resp_data = await self.solvable_connector.recv()
if not secrets.compare_digest(self.token, resp_token):
raise HTTPError(500, 'Solvable didn\'t provide initial token.')
log.debug('Received answer from solvable')
self.write(resp_data)

View File

@ -1,15 +0,0 @@
import secrets
from tornado.web import RequestHandler, HTTPError
class TestHandler(RequestHandler):
def initialize(self, solvable_connector, token):
self.solvable_connector = solvable_connector
self.token = token
async def get(self):
self.solvable_connector.send('test', {})
resp_token, resp_data = await self.solvable_connector.recv()
if not secrets.compare_digest(self.token, resp_token):
raise HTTPError(500, 'Solvable didn\'t provide initial token.')
self.write(resp_data)

View File

@ -1,21 +0,0 @@
import sys
import tornado
import zmq
from tornado.ioloop import IOLoop
from sql_injection_fsm import SQLInjectionFSM
from tfw.networking.server.tfw_server import TFWServer
from tfw.config import tfwenv
from tfw.config.logs import logging
log = logging.getLogger(__name__)
if __name__ == '__main__':
server = TFWServer(SQLInjectionFSM)
server.listen(tfwenv.WEB_PORT)
log.debug('Python version: {}'.format(sys.version[:5]))
log.debug('Tornado version: {}'.format(tornado.version))
log.debug('ZeroMQ version: {}'.format(zmq.zmq_version()))
log.debug('PyZMQ version: {}'.format(zmq.pyzmq_version()))
log.info('Tornado application listening on port {}'.format(tfwenv.WEB_PORT))
IOLoop.instance().start()

View File

@ -1,54 +0,0 @@
from tornado.ioloop import IOLoop
from tfw.components.source_code_event_handler import SourceCodeEventHandler
from tfw.components.terminado_event_handler import TerminadoEventHandler
from tfw.components.process_managing_event_handler import ProcessManagingEventHandler
from tfw.config import tfwenv
from tfw.message_sender import MessageSender
from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
from tfw.config.logs import logging
log = logging.getLogger(__name__)
def cenator(history):
log.debug('User executed command: "{}"'.format(history[-1]))
MessageSender().send('JOHN CENA', 'You\'ve executed "{}"'.format(history[-1]))
def selectdir(history):
try:
cmd = history[-1].split()
if cmd[0] == 'selectdir':
ServerUplinkConnector().send_to_eventhandler('webide',
{'data': {'command': 'selectdir',
'directory': cmd[1]}})
except Exception:
log.exception('Selectdir failed!')
def toggle_next(history):
toggle_next.button_state = not toggle_next.button_state
try:
cmd = history[-1].split()
if cmd[0] == 'togglenext':
ServerUplinkConnector().send('messagecontrol',
{'data': {'command': 'showbutton',
'next_visibility': toggle_next.button_state}})
except Exception:
log.exception('Togglenext failed!')
toggle_next.button_state = False
if __name__ == '__main__':
ide = SourceCodeEventHandler(key='webide', directory=tfwenv.WEBIDE_WD, exclude=['*.pyc'])
terminado = TerminadoEventHandler(key='shell')
terminado.historymonitor.subscribe_callback(cenator)
terminado.historymonitor.subscribe_callback(selectdir)
terminado.historymonitor.subscribe_callback(toggle_next)
processmanager = ProcessManagingEventHandler(key='processmanager', dirmonitor=ide.monitor)
eventhandlers = {ide, terminado, processmanager}
try:
IOLoop.instance().start()
finally:
for eh in eventhandlers: eh.cleanup()

View File

@ -1,27 +0,0 @@
import sqlite3
def get_db():
return sqlite3.connect('users.db')
def authorize_login(email, password):
"""
This method checks if a user is authorized and has admin privileges.
:param email: The email address of the user.
:param password: The password of the user.
:return: A tuple, the first element is the email address if the user exists,
and None if they don't; the second element is a boolean, which is True if
the user has admin privileges.
"""
conn = get_db()
sql_statement = '''SELECT email, is_admin FROM users
WHERE email="{}" AND password="{}"'''
# The problem with this approach is that it substitutes any value received
# from the user, even if it is a valid SQL statement!
result = conn.execute(sql_statement.format(email, password)).fetchone()
if result is None:
return None, False
else:
email, is_admin = result
return email, is_admin == 1

View File

@ -1,27 +0,0 @@
import json, sys
from tornado.ioloop import IOLoop
from tornado.web import RequestHandler, Application
from tfw.config import tfwenv
sys.path.append(tfwenv.WEBIDE_WD)
from login_component import authorize_login
class LoginHandler(RequestHandler):
def post(self, *args, **kwargs):
request = json.loads(self.request.body)
email, is_admin = authorize_login(
request['email'],
request['password']
)
self.write({
'email': email,
'is_admin': is_admin
})
if __name__ == '__main__':
application = Application([(r'/login', LoginHandler)])
application.listen(tfwenv.LOGIN_APP_PORT)
IOLoop.instance().start()

Binary file not shown.

View File

@ -1,26 +0,0 @@
from tfw.fsm_base import FSMBase
class SQLInjectionFSM(FSMBase):
states = [
'start',
'stripped_code',
'sql',
'commented_code',
'sql_with_substitutions',
'sql_output',
'end',
]
transitions = [
{'trigger': 'webide', 'source': '*', 'dest': 'stripped_code'}, # TODO: delet this
{'trigger': 'webide', 'source': 'start', 'dest': 'stripped_code'},
{'trigger': 'login', 'source': 'stripped_code', 'dest': 'sql'},
{'trigger': 'logger', 'source': 'sql', 'dest': 'commented_code'},
{'trigger': 'webide', 'source': 'commented_code', 'dest': 'sql_with_substitutions'},
{'trigger': 'logger', 'source': 'sql_with_substitutions', 'dest': 'sql_output'},
{'trigger': 'logger', 'source': 'sql_output', 'dest': 'end'},
{'trigger': 'reset', 'source': 'end', 'dest': 'start'},
]
def __init__(self):
super().__init__('start')

@ -1 +0,0 @@
Subproject commit 792ad68f8acd71971fb14326477b6925dda927c5

View File

@ -1,3 +0,0 @@
[program:controller]
directory=%(ENV_TFW_CONTROLLER_DIR)s
command=env python app.py

View File

@ -5,14 +5,15 @@ set -e
SCRIPT_DIR="$(dirname $($readlink_cmd -f $0))" SCRIPT_DIR="$(dirname $($readlink_cmd -f $0))"
TAO_PATH="${TAO_PATH:-$SCRIPT_DIR/..}" TAO_PATH="${TAO_PATH:-$SCRIPT_DIR/..}"
BACKEND_REPO="${BACKEND_REPO:-tutorial-framework-wip}" BACKEND_REPO="${BACKEND_REPO:-test-tutorial-framework}"
FRONTEND_REPO="${FRONTEND_REPO:-tutorial-framework-ng}" FRONTEND_REPO="${FRONTEND_REPO:-frontend-tutorial-framework}"
BACKEND_PATH="${TAO_PATH}/${BACKEND_REPO}" BACKEND_PATH="${TAO_PATH}/${BACKEND_REPO}"
FRONTEND_PATH="${TAO_PATH}/${FRONTEND_REPO}" FRONTEND_PATH="${TAO_PATH}/${FRONTEND_REPO}"
IMAGE_NAME="${IMAGE_NAME:-tfw}" IMAGE_NAME="${IMAGE_NAME:-baseimage-tutorial-framework}"
BACKEND_PORT="${BACKEND_PORT:-8888}" BACKEND_PORT="${BACKEND_PORT:-8888}"
AVATAO_SECRET="${AVATAO_SECRET:-secret}" AVATAO_SECRET="${AVATAO_SECRET:-secret}"
BUILD_CONTEXT="${BUILD_CONTEXT:-solvable}"
function run_frontend() function run_frontend()
{ {
@ -23,7 +24,7 @@ function run_frontend()
function run_backend() function run_backend()
{ {
cd $BACKEND_PATH cd $BACKEND_PATH
docker build -t $IMAGE_NAME . docker build -t $IMAGE_NAME -f ${BUILD_CONTEXT}/Dockerfile --build-arg BUILD_CONTEXT=$BUILD_CONTEXT --build-arg NOFRONTEND=1 .
docker run --rm -p $BACKEND_PORT:$BACKEND_PORT -e AVATAO_SECRET=$AVATAO_SECRET $IMAGE_NAME docker run --rm -p $BACKEND_PORT:$BACKEND_PORT -e AVATAO_SECRET=$AVATAO_SECRET $IMAGE_NAME
} }