Merge pull request #24 from avatao-content/baseimage_final

Refactor project into TFW baseimage

Dockerfile

@@ -34,21 +34,15 @@ RUN curl -fSL -o pyenv-installer ${PYENV_INSTALLER_URL}              &&\
     pip install -r /tmp/requirements.txt
 
 USER root
-COPY src/frontend /data/
-RUN cd /data && yarn install --frozen-lockfile
-RUN cd /data && yarn build --no-progress
-
 ENV TFW_PUBLIC_PORT=8888          \
     TFW_WEB_PORT=4242             \
     TFW_LOGIN_APP_PORT=6666       \
     TFW_TERMINADO_PORT=7878       \
     TFW_SUPERVISOR_HTTP_PORT=9001 \
-    TFW_CONTROLLER_PORT=7777      \
-    TFW_CRP_LISTENER_PORT=5555    \
     TFW_PUBLISHER_PORT=7654       \
     TFW_RECEIVER_PORT=8765
 
-EXPOSE ${TFW_PUBLIC_PORT} ${TFW_CRP_LISTENER_PORT}
+EXPOSE ${TFW_PUBLIC_PORT}
 
 ENV PYTHONPATH="/usr/local/lib/"                                           \
     TFW_SUPERVISOR_HTTP_URI="http://localhost:${TFW_SUPERVISOR_HTTP_PORT}" \
@@ -58,7 +52,6 @@ ENV PYTHONPATH="/usr/local/lib/"                                           \
     TFW_NGINX_DEFAULT="/etc/nginx/sites-enabled/default"                   \
     TFW_NGINX_COMPONENTS="/etc/nginx/components"                           \
     TFW_LIB_DIR="/usr/local/lib/"                                          \
-    TFW_CONTROLLER_DIR="/srv/controller"                                   \
     TFW_TERMINADO_DIR="/tmp/terminado_server"                              \
     TFW_FRONTEND_DIR="/srv/frontend"                                       \
     TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history"                      \
@@ -73,38 +66,25 @@ RUN echo "shopt -s cmdhist\n"                 \
          'PROMPT_COMMAND="history -a"\n'      \
     >> /home/${AVATAO_USER}/.bashrc
 
+COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF}
 COPY nginx/nginx.conf ${TFW_NGINX_CONF}
 COPY nginx/default.conf ${TFW_NGINX_DEFAULT}
-COPY nginx/components/ ${TFW_NGINX_COMPONENTS}
+COPY lib ${TFW_LIB_DIR}
-RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx                                          &&\
+
+ONBUILD ARG BUILD_CONTEXT="."
+ONBUILD ARG NOFRONTEND=""
+
+ONBUILD COPY ${BUILD_CONTEXT}/nginx/components/ ${TFW_NGINX_COMPONENTS}
+ONBUILD COPY ${BUILD_CONTEXT}/supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS}
+
+ONBUILD RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx                                          &&\
             for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do                               \
                 envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f~ && mv $f~ $f ;\
             done
 
-COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF}
+ONBUILD COPY ${BUILD_CONTEXT}/frontend /data/
-COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS}
+ONBUILD RUN if [ -z "${NOFRONTEND}" ]; then cd /data && yarn install --frozen-lockfile; fi
-COPY lib ${TFW_LIB_DIR}
+ONBUILD RUN if [ -z "${NOFRONTEND}" ]; then cd /data && yarn build --no-progress; fi
-COPY src/controller ${TFW_CONTROLLER_DIR}
+ONBUILD RUN if [ -z "${NOFRONTEND}" ]; then mv /data/dist ${TFW_FRONTEND_DIR} && rm -rf /data; fi
-RUN mv /data/dist ${TFW_FRONTEND_DIR} && rm -rf /data
-
-### TFW internals ^ ### DEMO v ###############################################################
-
-ENV TFW_APP_DIR="/srv/app"                          \
-    TFW_LOGIN_APP_DIR="/tmp/source_code_server"     \
-    TFW_WEBIDE_WD="/home/${AVATAO_USER}/workdir"    \
-    TFW_TERMINADO_WD="/home/${AVATAO_USER}/workdir"
-
-COPY src/demo ${TFW_APP_DIR}/
-
-COPY src/demo/source_code_server/server.py ${TFW_LOGIN_APP_DIR}/
-COPY src/demo/source_code_server/users.db ${TFW_LOGIN_APP_DIR}/
-COPY src/demo/source_code_server/login_component.py ${TFW_WEBIDE_WD}/
-
-RUN chown -R ${AVATAO_USER} ${TFW_WEBIDE_WD} &&\
-    chmod -R 755 ${TFW_WEBIDE_WD}
-
-USER ${AVATAO_USER}
-VOLUME ["/etc/nginx", "/home/${AVATAO_USER}", "/var/lib/nginx", "/var/log/nginx"]
-WORKDIR /home/${AVATAO_USER}
 
 CMD . "$HOME/.pyenvrc" && exec supervisord --nodaemon

README.md

@@ -5,17 +5,12 @@ Tutorial Framework
 
 Execute the `tfw_magic_start.sh` script and it will automagically run a backend instance in Docker and serve the Angular frontend locally.
 
-This requires the `tutorial-framework-wip` and `tutorial-framework-ng` repositories to have a common parent folder.
+This requires the `test-tutorial-framework` (Docker image `FROM baseimage-tutorial-framework`) and `frontend-tutorial-framework` repositories to have a common parent folder.
 
-## Building and running with Docker
+## Building with Docker
 
-Simply issue `docker build -t tfw .` in the project root. The first build could take a while as it's compiling a fresh
+Simply issue `docker build -t baseimage-tutorial-framework .` in the project root.
-Python package from source.
 
-Run with `docker run --rm -p 8888:8888 -e AVATAO_SECRET=secret tfw`.
+## Creating child images
 
-Running locally is possible with lots of pain involved, so this is no longer officially supported.
+Documentation in progress...
-
-## Frontend
-
-Place an `index.html` and related static files in `src/frontend/dist`, the web server will serve them.

lib/tfw/networking/async_solvable_connector.py

@@ -1,14 +0,0 @@
-from tfw.networking.serialization import deserialize_all
-from tfw.networking.solvable_connector import SolvableConnector
-
-
-class AsyncSolvableConnector(SolvableConnector):
-    def __init__(self, async_zmq_context=None):
-        if async_zmq_context is None:
-            from zmq.eventloop.future import Context
-            async_zmq_context = Context.instance()
-        super(AsyncSolvableConnector, self).__init__(async_zmq_context)
-
-    async def recv(self):
-        response = await self._zmq_req_socket.recv_multipart()
-        return deserialize_all(*response)

lib/tfw/networking/server/tfw_server.py

@@ -1,7 +1,6 @@
 from tornado.web import Application
 from collections import defaultdict
 
-from tfw.networking.server.controller_responder import ControllerResponder
 from tfw.networking.server.zmq_websocket_handler import ZMQWebSocketProxy
 from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
 from tfw.message_sender import MessageSender
@@ -20,7 +19,7 @@ class TFWServer:
             [(r'/ws', ZMQWebSocketProxy, {'make_response': self.make_response,
                                           'proxy_filter': self.proxy_filter})]
         )
-        self.controller_responder = ControllerResponder(self.fsm)
+        #self.controller_responder = ControllerResponder(self.fsm) TODO: add this once controller stuff is resolved
 
     @property
     def fsm(self):

lib/tfw/networking/solvable_connector.py

@@ -1,18 +0,0 @@
-import zmq
-
-from tfw.config import tfwenv
-from tfw.networking.zmq_connector_base import ZMQConnectorBase
-from tfw.networking.serialization import serialize_all, deserialize_all
-
-
-class SolvableConnector(ZMQConnectorBase):
-    def __init__(self, zmq_context=None):
-        super(SolvableConnector, self).__init__(zmq_context)
-        self._zmq_req_socket = self._zmq_context.socket(zmq.REQ)
-        self._zmq_req_socket.bind('tcp://*:{}'.format(tfwenv.CONTROLLER_PORT))
-
-    def send(self, key, message):
-        self._zmq_req_socket.send_multipart(serialize_all(key, message))
-
-    def recv(self):
-        return deserialize_all(*self._zmq_req_socket.recv_multipart())

src/controller/app.py

@@ -1,38 +0,0 @@
-import secrets
-from tornado.ioloop import IOLoop
-from tornado.web import Application
-
-from tao.config import taoenv
-from tfw.config import tfwenv
-from handlers import SolutionCheckHandler, TestHandler
-from tfw.networking.async_solvable_connector import AsyncSolvableConnector
-
-from tfw.config.logs import logging
-
-log = logging.getLogger(__name__)
-
-
-async def setup_token(solvable_connector, token):
-    solvable_connector.send('token', token)
-    key, data = await solvable_connector.recv()
-    if secrets.compare_digest(token, key):
-        log.debug('Token setup is done.')
-    else:
-        log.error('Failed to setup token between controller and solvable.')  # TODO: signaling to Avatao platform?
-
-if __name__ == '__main__':
-    kwargs = {
-        'solvable_connector': AsyncSolvableConnector(),
-        'token': secrets.token_hex(32)
-    }
-    IOLoop.instance().add_callback(setup_token, **kwargs)
-    routes = [
-        (r'/{secret}/?'.format(secret=taoenv.SECRET), SolutionCheckHandler, kwargs),
-        (r'/{secret}/test/?'.format(secret=taoenv.SECRET), TestHandler, kwargs)
-    ]
-    app = Application(
-        routes
-    )
-    app.listen(tfwenv.CRP_LISTENER_PORT)
-    log.debug('Controller listening on {}'.format(tfwenv.CRP_LISTENER_PORT))
-    IOLoop.instance().start()

src/controller/handlers/__init__.py

@@ -1,2 +0,0 @@
-from .solution_check_handler import SolutionCheckHandler
-from .test_handler import TestHandler

src/controller/handlers/solution_check_handler.py

@@ -1,19 +0,0 @@
-import secrets
-from tornado.web import RequestHandler, HTTPError
-
-from tfw.config.logs import logging
-log = logging.getLogger(__name__)
-
-class SolutionCheckHandler(RequestHandler):
-    def initialize(self, solvable_connector, token):
-        self.solvable_connector = solvable_connector
-        self.token = token
-
-    async def get(self):
-        log.debug('Sending request to solvable')
-        self.solvable_connector.send('solution_check', {})
-        resp_token, resp_data = await self.solvable_connector.recv()
-        if not secrets.compare_digest(self.token, resp_token):
-            raise HTTPError(500, 'Solvable didn\'t provide initial token.')
-        log.debug('Received answer from solvable')
-        self.write(resp_data)

src/controller/handlers/test_handler.py

@@ -1,15 +0,0 @@
-import secrets
-from tornado.web import RequestHandler, HTTPError
-
-
-class TestHandler(RequestHandler):
-    def initialize(self, solvable_connector, token):
-        self.solvable_connector = solvable_connector
-        self.token = token
-
-    async def get(self):
-        self.solvable_connector.send('test', {})
-        resp_token, resp_data = await self.solvable_connector.recv()
-        if not secrets.compare_digest(self.token, resp_token):
-            raise HTTPError(500, 'Solvable didn\'t provide initial token.')
-        self.write(resp_data)

src/demo/app.py

@@ -1,21 +0,0 @@
-import sys
-import tornado
-import zmq
-from tornado.ioloop import IOLoop
-
-from sql_injection_fsm import SQLInjectionFSM
-from tfw.networking.server.tfw_server import TFWServer
-from tfw.config import tfwenv
-from tfw.config.logs import logging
-log = logging.getLogger(__name__)
-
-
-if __name__ == '__main__':
-    server = TFWServer(SQLInjectionFSM)
-    server.listen(tfwenv.WEB_PORT)
-    log.debug('Python version: {}'.format(sys.version[:5]))
-    log.debug('Tornado version: {}'.format(tornado.version))
-    log.debug('ZeroMQ version: {}'.format(zmq.zmq_version()))
-    log.debug('PyZMQ version: {}'.format(zmq.pyzmq_version()))
-    log.info('Tornado application listening on port {}'.format(tfwenv.WEB_PORT))
-    IOLoop.instance().start()

src/demo/event_handler_main.py

@@ -1,54 +0,0 @@
-from tornado.ioloop import IOLoop
-
-from tfw.components.source_code_event_handler import SourceCodeEventHandler
-from tfw.components.terminado_event_handler import TerminadoEventHandler
-from tfw.components.process_managing_event_handler import ProcessManagingEventHandler
-from tfw.config import tfwenv
-from tfw.message_sender import MessageSender
-from tfw.networking.event_handlers.server_connector import ServerUplinkConnector
-from tfw.config.logs import logging
-log = logging.getLogger(__name__)
-
-
-def cenator(history):
-    log.debug('User executed command: "{}"'.format(history[-1]))
-    MessageSender().send('JOHN CENA', 'You\'ve executed "{}"'.format(history[-1]))
-
-
-def selectdir(history):
-    try:
-        cmd = history[-1].split()
-        if cmd[0] == 'selectdir':
-            ServerUplinkConnector().send_to_eventhandler('webide',
-                                                         {'data': {'command': 'selectdir',
-                                                                   'directory': cmd[1]}})
-    except Exception:
-        log.exception('Selectdir failed!')
-
-
-def toggle_next(history):
-    toggle_next.button_state = not toggle_next.button_state
-    try:
-        cmd = history[-1].split()
-        if cmd[0] == 'togglenext':
-            ServerUplinkConnector().send('messagecontrol',
-                                         {'data': {'command': 'showbutton',
-                                                   'next_visibility': toggle_next.button_state}})
-    except Exception:
-        log.exception('Togglenext failed!')
-toggle_next.button_state = False
-
-
-if __name__ == '__main__':
-    ide = SourceCodeEventHandler(key='webide', directory=tfwenv.WEBIDE_WD, exclude=['*.pyc'])
-    terminado = TerminadoEventHandler(key='shell')
-    terminado.historymonitor.subscribe_callback(cenator)
-    terminado.historymonitor.subscribe_callback(selectdir)
-    terminado.historymonitor.subscribe_callback(toggle_next)
-    processmanager = ProcessManagingEventHandler(key='processmanager', dirmonitor=ide.monitor)
-
-    eventhandlers = {ide, terminado, processmanager}
-    try:
-        IOLoop.instance().start()
-    finally:
-        for eh in eventhandlers: eh.cleanup()

src/demo/source_code_server/login_component.py

@@ -1,27 +0,0 @@
-import sqlite3
-
-
-def get_db():
-    return sqlite3.connect('users.db')
-
-
-def authorize_login(email, password):
-    """
-    This method checks if a user is authorized and has admin privileges.
-    :param email: The email address of the user.
-    :param password: The password of the user.
-    :return: A tuple, the first element is the email address if the user exists,
-    and None if they don't; the second element is a boolean, which is True if
-    the user has admin privileges.
-    """
-    conn = get_db()
-    sql_statement = '''SELECT email, is_admin FROM users
-                       WHERE email="{}" AND password="{}"'''
-    # The problem with this approach is that it substitutes any value received
-    # from the user, even if it is a valid SQL statement!
-    result = conn.execute(sql_statement.format(email, password)).fetchone()
-    if result is None:
-        return None, False
-    else:
-        email, is_admin = result
-        return email, is_admin == 1

src/demo/source_code_server/server.py

@@ -1,27 +0,0 @@
-import json, sys
-from tornado.ioloop import IOLoop
-from tornado.web import RequestHandler, Application
-
-from tfw.config import tfwenv
-
-sys.path.append(tfwenv.WEBIDE_WD)
-from login_component import authorize_login
-
-
-class LoginHandler(RequestHandler):
-    def post(self, *args, **kwargs):
-        request = json.loads(self.request.body)
-        email, is_admin = authorize_login(
-            request['email'],
-            request['password']
-        )
-        self.write({
-            'email': email,
-            'is_admin': is_admin
-        })
-
-
-if __name__ == '__main__':
-    application = Application([(r'/login', LoginHandler)])
-    application.listen(tfwenv.LOGIN_APP_PORT)
-    IOLoop.instance().start()

src/demo/sql_injection_fsm.py

@@ -1,26 +0,0 @@
-from tfw.fsm_base import FSMBase
-
-
-class SQLInjectionFSM(FSMBase):
-    states = [
-        'start',
-        'stripped_code',
-        'sql',
-        'commented_code',
-        'sql_with_substitutions',
-        'sql_output',
-        'end',
-    ]
-    transitions = [
-        {'trigger': 'webide', 'source': '*', 'dest': 'stripped_code'},  # TODO: delet this
-        {'trigger': 'webide', 'source': 'start', 'dest': 'stripped_code'},
-        {'trigger': 'login', 'source': 'stripped_code', 'dest': 'sql'},
-        {'trigger': 'logger', 'source': 'sql', 'dest': 'commented_code'},
-        {'trigger': 'webide', 'source': 'commented_code', 'dest': 'sql_with_substitutions'},
-        {'trigger': 'logger', 'source': 'sql_with_substitutions', 'dest': 'sql_output'},
-        {'trigger': 'logger', 'source': 'sql_output', 'dest': 'end'},
-        {'trigger': 'reset', 'source': 'end', 'dest': 'start'},
-    ]
-
-    def __init__(self):
-        super().__init__('start')

src/frontend

@@ -1 +0,0 @@
-Subproject commit 792ad68f8acd71971fb14326477b6925dda927c5

supervisor/components/controller.conf

@@ -1,3 +0,0 @@
-[program:controller]
-directory=%(ENV_TFW_CONTROLLER_DIR)s
-command=env python app.py

tfw_magic_start.sh

@@ -5,14 +5,15 @@ set -e
 SCRIPT_DIR="$(dirname $($readlink_cmd -f $0))"
 
 TAO_PATH="${TAO_PATH:-$SCRIPT_DIR/..}"
-BACKEND_REPO="${BACKEND_REPO:-tutorial-framework-wip}"
+BACKEND_REPO="${BACKEND_REPO:-test-tutorial-framework}"
-FRONTEND_REPO="${FRONTEND_REPO:-tutorial-framework-ng}"
+FRONTEND_REPO="${FRONTEND_REPO:-frontend-tutorial-framework}"
 BACKEND_PATH="${TAO_PATH}/${BACKEND_REPO}"
 FRONTEND_PATH="${TAO_PATH}/${FRONTEND_REPO}"
 
-IMAGE_NAME="${IMAGE_NAME:-tfw}"
+IMAGE_NAME="${IMAGE_NAME:-baseimage-tutorial-framework}"
 BACKEND_PORT="${BACKEND_PORT:-8888}"
 AVATAO_SECRET="${AVATAO_SECRET:-secret}"
+BUILD_CONTEXT="${BUILD_CONTEXT:-solvable}"
 
 function run_frontend()
 {
@@ -23,7 +24,7 @@ function run_frontend()
 function run_backend()
 {
     cd $BACKEND_PATH
-    docker build -t $IMAGE_NAME .
+    docker build -t $IMAGE_NAME -f ${BUILD_CONTEXT}/Dockerfile --build-arg BUILD_CONTEXT=$BUILD_CONTEXT --build-arg NOFRONTEND=1 .
     docker run --rm -p $BACKEND_PORT:$BACKEND_PORT -e AVATAO_SECRET=$AVATAO_SECRET $IMAGE_NAME
 }