From fa115a67c006e79d6a806f77cbdc2ef019161ba9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 08:48:21 +0100 Subject: [PATCH 01/16] Fix envsubst undefined behaviour on some platforms in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cd0ce9f..72460b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,7 +78,7 @@ COPY nginx/components/ ${TFW_NGINX_COMPONENTS} RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\ sed -i 's#pid /run/nginx.pid;#pid /tmp/nginx.pid;#g' /etc/nginx/nginx.conf &&\ for f in "${TFW_NGINX_CONF}" ${TFW_NGINX_COMPONENTS}/*.conf; do \ - envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f ;\ + envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f~ && mv $f~ $f ;\ done COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF} From 51234c750c231e23a9d37a8913d0bd439da62a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 08:57:05 +0100 Subject: [PATCH 02/16] Add nginx permission fixes to Dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 72460b2..5b45389 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,7 +101,8 @@ COPY src/demo/source_code_server/server.py ${TFW_LOGIN_APP_DIR}/ COPY src/demo/source_code_server/users.db ${TFW_LOGIN_APP_DIR}/ COPY src/demo/source_code_server/login_component.py ${TFW_WEBIDE_WD}/ -RUN chown -R ${AVATAO_USER} ${TFW_WEBIDE_WD} && chmod -R 755 ${TFW_WEBIDE_WD} +RUN chown -R ${AVATAO_USER} ${TFW_WEBIDE_WD} /var/log/nginx /var/lib/nginx /etc/nginx &&\ + chmod -R 755 ${TFW_WEBIDE_WD} USER ${AVATAO_USER} WORKDIR /home/${AVATAO_USER} From 8cd0665c03383ea651d962e14fddf2a92005bf2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 08:57:33 +0100 Subject: [PATCH 03/16] Make rw directories VOLUMEs --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 5b45389..a1dff20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -105,6 +105,7 @@ RUN chown -R ${AVATAO_USER} ${TFW_WEBIDE_WD} /var/log/nginx /var/lib/nginx /etc/ chmod -R 755 ${TFW_WEBIDE_WD} USER ${AVATAO_USER} +VOLUME ["/etc/nginx", "/home/${AVATAO_USER}", "/var/lib/nginx", "/var/log/nginx"] WORKDIR /home/${AVATAO_USER} CMD . "$HOME/.pyenvrc" && exec supervisord --nodaemon From 49b728ee6c9562217b2878b85cd81130e27acd6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 09:01:03 +0100 Subject: [PATCH 04/16] Rename nginx default site config --- Dockerfile | 2 +- nginx/{nginx.conf => default.conf} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename nginx/{nginx.conf => default.conf} (100%) diff --git a/Dockerfile b/Dockerfile index a1dff20..bd9a738 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,7 +73,7 @@ RUN echo "shopt -s cmdhist\n" \ 'PROMPT_COMMAND="history -a"\n' \ >> /home/${AVATAO_USER}/.bashrc -COPY nginx/nginx.conf ${TFW_NGINX_CONF} +COPY nginx/default.conf ${TFW_NGINX_CONF} COPY nginx/components/ ${TFW_NGINX_COMPONENTS} RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\ sed -i 's#pid /run/nginx.pid;#pid /tmp/nginx.pid;#g' /etc/nginx/nginx.conf &&\ diff --git a/nginx/nginx.conf b/nginx/default.conf similarity index 100% rename from nginx/nginx.conf rename to nginx/default.conf From 3caf170c7df9c8edef2c1da673e038ee46bd0692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 09:06:28 +0100 Subject: [PATCH 05/16] Add readonly-fs compatible configuration for nginx --- Dockerfile | 12 +++++++----- nginx/nginx.conf | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 nginx/nginx.conf diff --git a/Dockerfile b/Dockerfile index bd9a738..32e4a9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,7 +55,8 @@ ENV PYTHONPATH="/usr/local/lib/" \ TFW_SUPERVISOR_HTTP_URI="http://localhost:${TFW_SUPERVISOR_HTTP_PORT}" \ TFW_SUPERVISORD_CONF="/etc/supervisor/supervisord.conf" \ TFW_SUPERVISORD_COMPONENTS="/etc/supervisor/conf" \ - TFW_NGINX_CONF="/etc/nginx/sites-enabled/default" \ + TFW_NGINX_CONF="/etc/nginx/nginx.conf" \ + TFW_NGINX_DEFAULT="/etc/nginx/sites-enabled/default" \ TFW_NGINX_COMPONENTS="/etc/nginx/components" \ TFW_LIB_DIR="/usr/local/lib/" \ TFW_CONTROLLER_DIR="/srv/controller" \ @@ -73,11 +74,12 @@ RUN echo "shopt -s cmdhist\n" \ 'PROMPT_COMMAND="history -a"\n' \ >> /home/${AVATAO_USER}/.bashrc -COPY nginx/default.conf ${TFW_NGINX_CONF} +COPY nginx/nginx.conf ${TFW_NGINX_CONF} +COPY nginx/default.conf ${TFW_NGINX_DEFAULT} COPY nginx/components/ ${TFW_NGINX_COMPONENTS} -RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\ - sed -i 's#pid /run/nginx.pid;#pid /tmp/nginx.pid;#g' /etc/nginx/nginx.conf &&\ - for f in "${TFW_NGINX_CONF}" ${TFW_NGINX_COMPONENTS}/*.conf; do \ +RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\ + sed -i 's#pid /run/nginx.pid;#pid /tmp/nginx.pid;#g' /etc/nginx/nginx.conf &&\ + for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do \ envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f~ && mv $f~ $f ;\ done diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..5d1e5ec --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,24 @@ +worker_processes auto; +pid /tmp/nginx.pid; + +events +{ + worker_connections 1024; +} + +http +{ + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + access_log /dev/stdout; + error_log /dev/stderr; + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} From 3e0592c17684900e15966ee9ebbe98fc228415e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 09:25:42 +0100 Subject: [PATCH 06/16] Remove Cenator :'( sorry to see you go --- lib/tfw/event_handler_base.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base.py index 93172ce..768c1f6 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base.py @@ -1,18 +1,7 @@ -from itertools import cycle - from tfw.networking.serialization import deserialize_all from tfw.networking.event_handlers.server_connector import ServerConnector -def cenator(): - yield from [ - '**Your time is up**', - '`My time is now`', - '_You can\'t see me_', - 'My time is now' - ] - - class EventHandlerBase: def __init__(self, key): self.server_connector = ServerConnector() @@ -21,13 +10,9 @@ class EventHandlerBase: self.subscribe(self.key) self.subscribe('reset') self.server_connector.register_callback(self.event_handler_callback) - self.cenerator = cycle(cenator()) def event_handler_callback(self, msg_parts): key, message = deserialize_all(*msg_parts) - from .message_sender import MessageSender - ms = MessageSender() - ms.send('avataobot', next(self.cenerator)) response = self.dispatch_handling(key, message) if response is None: return self.server_connector.send(key, response) From 306607c972ef80afdff9dd5b8a5df29fe0a92c6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 09:30:44 +0100 Subject: [PATCH 07/16] =?UTF-8?q?Reintroduce=20glorious=20JOHN=20CENA=20to?= =?UTF-8?q?=20demo=20=C2=AF\=5F(=E3=83=84)=5F/=C2=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/demo/event_handler_main.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/demo/event_handler_main.py b/src/demo/event_handler_main.py index 6cb9c8a..69d2fd8 100644 --- a/src/demo/event_handler_main.py +++ b/src/demo/event_handler_main.py @@ -4,14 +4,21 @@ from tfw.components.source_code_event_handler import SourceCodeEventHandler from tfw.components.terminado_event_handler import TerminadoEventHandler from tfw.components.process_managing_event_handler import ProcessManagingEventHandler from tfw.config import tfwenv +from tfw.message_sender import MessageSender from tfw.config.logs import logging log = logging.getLogger(__name__) +def cenator(history): + message_sender = MessageSender() + log.debug('User executed command: "{}"'.format(history[-1])) + message_sender.send('JOHN CENA', 'You\'ve executed "{}"'.format(history[-1])) + + if __name__ == '__main__': ide = SourceCodeEventHandler(key='webide', directory=tfwenv.WEBIDE_WD, exclude=['__pycache__']) terminado = TerminadoEventHandler(key='shell') - terminado.historymonitor.subscribe_callback(callback=lambda hist: log.debug('User executed command: "{}"'.format(hist[-1]))) + terminado.historymonitor.subscribe_callback(cenator) processmanager = ProcessManagingEventHandler(key='processmanager', dirmonitor=ide.monitor) eventhandlers = {ide, terminado, processmanager} From e080fb0e5c1c0d9d0860800177ba2d48314457be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 10:23:22 +0100 Subject: [PATCH 08/16] Remove lefover frontend source code from final image --- Dockerfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 32e4a9c..9616536 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,10 +34,9 @@ RUN curl -fSL -o pyenv-installer ${PYENV_INSTALLER_URL} &&\ pip install -r /tmp/requirements.txt USER root -WORKDIR /data/ COPY src/frontend /data/ -RUN yarn install --frozen-lockfile -RUN yarn build --no-progress +RUN cd /data && yarn install --frozen-lockfile +RUN cd /data && yarn build --no-progress ENV TFW_PUBLIC_PORT=8888 \ TFW_WEB_PORT=4242 \ @@ -87,6 +86,7 @@ COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF} COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS} COPY lib ${TFW_LIB_DIR} COPY src/controller ${TFW_CONTROLLER_DIR} +RUN mv /data/dist ${TFW_FRONTEND_DIR} && rm -rf /data ### TFW internals ^ ### DEMO v ############################################################### @@ -95,8 +95,6 @@ ENV TFW_APP_DIR="/srv/app" \ TFW_WEBIDE_WD="/home/${AVATAO_USER}/workdir" \ TFW_TERMINADO_WD="/home/${AVATAO_USER}/workdir" -RUN mv /data/dist ${TFW_FRONTEND_DIR} - COPY src/demo ${TFW_APP_DIR}/ COPY src/demo/source_code_server/server.py ${TFW_LOGIN_APP_DIR}/ From 7ae3180aba86fd5888d78f919ee0c9e69ef49dc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 10:23:46 +0100 Subject: [PATCH 09/16] Fix nginx config for Angular --- nginx/default.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nginx/default.conf b/nginx/default.conf index ea60388..563bced 100644 --- a/nginx/default.conf +++ b/nginx/default.conf @@ -15,7 +15,8 @@ server { include ${TFW_NGINX_COMPONENTS}/*.conf; location / { + index index.html; + try_files $uri $uri/ /index.html =404; root ${TFW_FRONTEND_DIR}; - try_files $uri $uri/ index.html; } } From 0361092f37f685d251e7d9f1a724b829ac006630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 10:49:32 +0100 Subject: [PATCH 10/16] Simplify MessageSender usage in cenator --- src/demo/event_handler_main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/demo/event_handler_main.py b/src/demo/event_handler_main.py index 69d2fd8..5cdefdc 100644 --- a/src/demo/event_handler_main.py +++ b/src/demo/event_handler_main.py @@ -10,9 +10,8 @@ log = logging.getLogger(__name__) def cenator(history): - message_sender = MessageSender() log.debug('User executed command: "{}"'.format(history[-1])) - message_sender.send('JOHN CENA', 'You\'ve executed "{}"'.format(history[-1])) + MessageSender().send('JOHN CENA', 'You\'ve executed "{}"'.format(history[-1])) if __name__ == '__main__': From 69f03f761507c64454e0bb0ab2f708df21451991 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 14:08:48 +0100 Subject: [PATCH 11/16] Implement EventHandler command mirroring (EH -> EH communication) --- .../server/zmq_websocket_handler.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/tfw/networking/server/zmq_websocket_handler.py b/lib/tfw/networking/server/zmq_websocket_handler.py index de2932e..017040e 100644 --- a/lib/tfw/networking/server/zmq_websocket_handler.py +++ b/lib/tfw/networking/server/zmq_websocket_handler.py @@ -21,12 +21,23 @@ class ZMQWebSocketHandler(WebSocketHandler): log.debug('WebSocket connection initiated') self._event_handler_connector.register_callback(self.zmq_callback) - @staticmethod - def zmq_callback(msg_parts): + @classmethod + def zmq_callback(cls, msg_parts): + keyhandlers = {'mirror': cls.mirror} + key, data = deserialize_all(*msg_parts) log.debug('Received on pull socket: {}'.format(data)) - for instance in ZMQWebSocketHandler.instances: - instance.write_message(data) + if key not in keyhandlers: + for instance in cls.instances: + instance.write_message(data) + else: + try: keyhandlers[key](data['data']) + except KeyError: log.error('Invalid mirror message format! Ignoring.') + + @classmethod + def mirror(cls, data): + key = data['key'] + cls._event_handler_connector.send_message({'data': data}, key) def on_message(self, message): log.debug('Received on WebSocket: {}'.format(message)) From a5d0359ab86c1e053319e2cd6db95224a8ea2070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 14:09:30 +0100 Subject: [PATCH 12/16] Add helper method to ServerUplinkConnector for EH -> EH communication --- lib/tfw/networking/event_handlers/server_connector.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/tfw/networking/event_handlers/server_connector.py b/lib/tfw/networking/event_handlers/server_connector.py index 7c3fcb3..09290fb 100644 --- a/lib/tfw/networking/event_handlers/server_connector.py +++ b/lib/tfw/networking/event_handlers/server_connector.py @@ -25,6 +25,10 @@ class ServerUplinkConnector(ZMQConnectorBase): self._zmq_push_socket = self._zmq_context.socket(zmq.PUSH) self._zmq_push_socket.connect('tcp://localhost:{}'.format(tfwenv.RECEIVER_PORT)) + def send_to_eventhandler(self, key, response): + response['data']['key'] = key + self.send('mirror', response) + def send(self, key, response): response['key'] = key self._zmq_push_socket.send_multipart(serialize_all(key, response)) From d4d6bdad0fef320cb3d6ace50f6065ca0878a9ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 14:12:09 +0100 Subject: [PATCH 13/16] Implement custom TFW command demo --- src/demo/event_handler_main.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/demo/event_handler_main.py b/src/demo/event_handler_main.py index 5cdefdc..ad5e9ce 100644 --- a/src/demo/event_handler_main.py +++ b/src/demo/event_handler_main.py @@ -5,6 +5,7 @@ from tfw.components.terminado_event_handler import TerminadoEventHandler from tfw.components.process_managing_event_handler import ProcessManagingEventHandler from tfw.config import tfwenv from tfw.message_sender import MessageSender +from tfw.networking.event_handlers.server_connector import ServerUplinkConnector from tfw.config.logs import logging log = logging.getLogger(__name__) @@ -14,10 +15,21 @@ def cenator(history): MessageSender().send('JOHN CENA', 'You\'ve executed "{}"'.format(history[-1])) +def selectdir(history): + try: + cmd = history[-1].split() + if cmd[0] == 'selectdir': + ServerUplinkConnector().send_to_eventhandler('webide', + {'data': {'command': 'selectdir', + 'directory': cmd[1]}}) + except Exception: + log.exception('Selectdir failed!') + if __name__ == '__main__': ide = SourceCodeEventHandler(key='webide', directory=tfwenv.WEBIDE_WD, exclude=['__pycache__']) terminado = TerminadoEventHandler(key='shell') terminado.historymonitor.subscribe_callback(cenator) + terminado.historymonitor.subscribe_callback(selectdir) processmanager = ProcessManagingEventHandler(key='processmanager', dirmonitor=ide.monitor) eventhandlers = {ide, terminado, processmanager} From 929ea905c576149ff00bdcf0632d87b278eba065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 14:15:40 +0100 Subject: [PATCH 14/16] Improve error handling in webide backend --- lib/tfw/components/source_code_event_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/components/source_code_event_handler.py b/lib/tfw/components/source_code_event_handler.py index 4915887..5ff61e8 100644 --- a/lib/tfw/components/source_code_event_handler.py +++ b/lib/tfw/components/source_code_event_handler.py @@ -106,7 +106,7 @@ class SourceCodeEventHandler(TriggerlessEventHandler): except IndexError: data['content'] = 'No files in this directory :(' except EnvironmentError: - log.exception('Failed to select directory "{}"'.format(data['directory'])) + log.error('Failed to select directory "{}"'.format(data['directory'])) return data def attach_fileinfo(self, data): From 1ea238e1085495d9d8d1b17efe29769b2585e7d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 14:31:34 +0100 Subject: [PATCH 15/16] Remove duplicate nginx pidfile fix from Dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9616536..4284ab3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,7 +77,6 @@ COPY nginx/nginx.conf ${TFW_NGINX_CONF} COPY nginx/default.conf ${TFW_NGINX_DEFAULT} COPY nginx/components/ ${TFW_NGINX_COMPONENTS} RUN chown -R ${AVATAO_USER} /var/log/nginx /var/lib/nginx &&\ - sed -i 's#pid /run/nginx.pid;#pid /tmp/nginx.pid;#g' /etc/nginx/nginx.conf &&\ for f in "${TFW_NGINX_DEFAULT}" ${TFW_NGINX_COMPONENTS}/*.conf; do \ envsubst "$(printenv | cut -d= -f1 | grep TFW_ | sed -e 's/^/$/g')" < $f > $f~ && mv $f~ $f ;\ done From 98443dc588fc83cf5ee51f2f8419c3c882bef8c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 14 Mar 2018 14:32:49 +0100 Subject: [PATCH 16/16] Remove duplicate nginx chown from Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4284ab3..3f11de4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,7 +100,7 @@ COPY src/demo/source_code_server/server.py ${TFW_LOGIN_APP_DIR}/ COPY src/demo/source_code_server/users.db ${TFW_LOGIN_APP_DIR}/ COPY src/demo/source_code_server/login_component.py ${TFW_WEBIDE_WD}/ -RUN chown -R ${AVATAO_USER} ${TFW_WEBIDE_WD} /var/log/nginx /var/lib/nginx /etc/nginx &&\ +RUN chown -R ${AVATAO_USER} ${TFW_WEBIDE_WD} &&\ chmod -R 755 ${TFW_WEBIDE_WD} USER ${AVATAO_USER}