From 338e690842427b5522d112c52a83d2fcc4e23a0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 2 Mar 2018 16:18:47 +0100 Subject: [PATCH 01/28] Remove terminado from supervisor --- Dockerfile | 1 - supervisor/components/terminado.conf | 4 ---- 2 files changed, 5 deletions(-) delete mode 100644 supervisor/components/terminado.conf diff --git a/Dockerfile b/Dockerfile index 4147105..5772ee0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,7 +74,6 @@ COPY supervisor/supervisord.conf ${TFW_SUPERVISORD_CONF} COPY supervisor/components/ ${TFW_SUPERVISORD_COMPONENTS} COPY lib ${TFW_LIB_DIR} COPY src/controller ${TFW_CONTROLLER_DIR} -COPY lib/tfw/components/terminado_mini_server.py ${TFW_TERMINADO_DIR}/ ### TFW internals ^ ### DEMO v ############################################################### diff --git a/supervisor/components/terminado.conf b/supervisor/components/terminado.conf deleted file mode 100644 index 30c441d..0000000 --- a/supervisor/components/terminado.conf +++ /dev/null @@ -1,4 +0,0 @@ -[program:terminado] -directory=%(ENV_TFW_TERMINADO_DIR)s -command=env python terminado_mini_server.py -autostart=false From cc2c6009ddc47f0e8c396e49ef576031ed312562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 2 Mar 2018 16:23:19 +0100 Subject: [PATCH 02/28] Delegate ownership of terminado server to TerminadoEventHandler --- lib/tfw/components/terminado_event_handler.py | 10 +++++----- src/demo/event_handler_main.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/tfw/components/terminado_event_handler.py b/lib/tfw/components/terminado_event_handler.py index 5329e3c..9156f44 100644 --- a/lib/tfw/components/terminado_event_handler.py +++ b/lib/tfw/components/terminado_event_handler.py @@ -1,16 +1,16 @@ +from tfw.components.terminado_mini_server import TerminadoMiniServer from tfw.event_handler_base import TriggerlessEventHandler -from tfw.components.mixins import SupervisorMixin from tfw.config import tfwenv from tfw.config.logs import logging log = logging.getLogger(__name__) -class TerminadoEventHandler(TriggerlessEventHandler, SupervisorMixin): - def __init__(self, key, process_name): +class TerminadoEventHandler(TriggerlessEventHandler): + def __init__(self, key): super().__init__(key) self.working_directory = tfwenv.TERMINADO_DIR - self.process_name = process_name - self.start_process(self.process_name) + self.terminado_server = TerminadoMiniServer('/terminal', tfwenv.TERMINADO_PORT, tfwenv.TERMINADO_WD, ['bash']) + self.terminado_server.listen() def handle_event(self, key, data_json): log.debug('TerminadoEventHandler received event for key {}'.format(key)) diff --git a/src/demo/event_handler_main.py b/src/demo/event_handler_main.py index dfbd5bb..b7090d3 100644 --- a/src/demo/event_handler_main.py +++ b/src/demo/event_handler_main.py @@ -8,7 +8,7 @@ from tfw.config import tfwenv if __name__ == '__main__': ide = SourceCodeEventHandler('webide', tfwenv.WEBIDE_WD) - terminado = TerminadoEventHandler('terminado', 'terminado') + terminado = TerminadoEventHandler('terminado') processmanager = ProcessManagingEventHandler('processmanager', ide.monitor) eventhandlers = {ide, terminado, processmanager} From b7f9acd3b0ff81c0c3d7d8914927392a1186172a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 2 Mar 2018 17:05:01 +0100 Subject: [PATCH 03/28] Expose pty reference from TerminadoMiniServer --- lib/tfw/components/terminado_mini_server.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/tfw/components/terminado_mini_server.py b/lib/tfw/components/terminado_mini_server.py index 32576cc..08b0c7d 100644 --- a/lib/tfw/components/terminado_mini_server.py +++ b/lib/tfw/components/terminado_mini_server.py @@ -10,15 +10,20 @@ log = logging.getLogger(__name__) class TerminadoMiniServer: def __init__(self, url, port, workdir, shellcmd): self.port = port + self.term_manager = UniqueTermManager(shell_command=shellcmd, + term_settings={'cwd': workdir}) self.application = Application( [( url, TerminadoMiniServer.CORSTermSocket, - {'term_manager': UniqueTermManager(shell_command=shellcmd, - term_settings={'cwd': workdir})} + {'term_manager': self.term_manager} )] ) + @property + def pty(self): + return self.term_manager.terminal.ptyproc + class CORSTermSocket(TermSocket): def check_origin(self, origin): return True From bddc10269bff3eb3231c7c083543ce727a7edf6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 2 Mar 2018 18:01:00 +0100 Subject: [PATCH 04/28] Replace UniqueTermManager with SingleTermManager to support pty manipulation --- lib/tfw/components/terminado_mini_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tfw/components/terminado_mini_server.py b/lib/tfw/components/terminado_mini_server.py index 08b0c7d..c4ee51e 100644 --- a/lib/tfw/components/terminado_mini_server.py +++ b/lib/tfw/components/terminado_mini_server.py @@ -1,6 +1,6 @@ from tornado.ioloop import IOLoop from tornado.web import Application -from terminado import TermSocket, UniqueTermManager +from terminado import TermSocket, SingleTermManager from tfw.config import tfwenv from tfw.config.logs import logging @@ -10,7 +10,7 @@ log = logging.getLogger(__name__) class TerminadoMiniServer: def __init__(self, url, port, workdir, shellcmd): self.port = port - self.term_manager = UniqueTermManager(shell_command=shellcmd, + self.term_manager = SingleTermManager(shell_command=shellcmd, term_settings={'cwd': workdir}) self.application = Application( [( From c4c07a4c85c7092cc3df5efe9e29c83fb2845c52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Fri, 2 Mar 2018 18:02:44 +0100 Subject: [PATCH 05/28] Add proof of concept for terminal pty writing --- lib/tfw/components/terminado_event_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/tfw/components/terminado_event_handler.py b/lib/tfw/components/terminado_event_handler.py index 9156f44..cb29e4e 100644 --- a/lib/tfw/components/terminado_event_handler.py +++ b/lib/tfw/components/terminado_event_handler.py @@ -14,4 +14,5 @@ class TerminadoEventHandler(TriggerlessEventHandler): def handle_event(self, key, data_json): log.debug('TerminadoEventHandler received event for key {}'.format(key)) - # TODO: wat do? + self.terminado_server.pty.write('echo cica\n') + # TODO: design & implement pty.write API From 4873bbf421b1ad7a08997cc5013f88d884b67e59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 3 Mar 2018 14:14:44 +0100 Subject: [PATCH 06/28] Implement simple pty writing API --- lib/tfw/components/terminado_event_handler.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/tfw/components/terminado_event_handler.py b/lib/tfw/components/terminado_event_handler.py index cb29e4e..00a90c9 100644 --- a/lib/tfw/components/terminado_event_handler.py +++ b/lib/tfw/components/terminado_event_handler.py @@ -10,9 +10,13 @@ class TerminadoEventHandler(TriggerlessEventHandler): super().__init__(key) self.working_directory = tfwenv.TERMINADO_DIR self.terminado_server = TerminadoMiniServer('/terminal', tfwenv.TERMINADO_PORT, tfwenv.TERMINADO_WD, ['bash']) + self.commands = {'write': self.write} self.terminado_server.listen() def handle_event(self, key, data_json): - log.debug('TerminadoEventHandler received event for key {}'.format(key)) - self.terminado_server.pty.write('echo cica\n') - # TODO: design & implement pty.write API + log.debug('TerminadoEventHandler received event: {}'.format(data_json)) + data = data_json['data'] + self.commands[data['command']](data) + + def write(self, data): + self.terminado_server.pty.write(data['shellcmd']) From 6312e221175eebd72af03f3f538caabd06dacbfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 3 Mar 2018 14:47:55 +0100 Subject: [PATCH 07/28] Make TerminadoMiniServer compatible with an external IOLoop --- lib/tfw/components/terminado_mini_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/components/terminado_mini_server.py b/lib/tfw/components/terminado_mini_server.py index c4ee51e..26afe31 100644 --- a/lib/tfw/components/terminado_mini_server.py +++ b/lib/tfw/components/terminado_mini_server.py @@ -30,9 +30,9 @@ class TerminadoMiniServer: def listen(self): self.application.listen(self.port) - IOLoop.instance().start() if __name__ == '__main__': log.info('Terminado Mini Server listening on {}'.format(tfwenv.TERMINADO_PORT)) TerminadoMiniServer('/terminal', tfwenv.TERMINADO_PORT, tfwenv.TERMINADO_WD, ['bash']).listen() + IOLoop.instance().start() From eea6a418a81507f5208c0ab0a8e5f2a76800baaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 3 Mar 2018 17:15:21 +0100 Subject: [PATCH 08/28] Start work on HistoryMonitor to read command history --- lib/tfw/components/history_monitor.py | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 lib/tfw/components/history_monitor.py diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py new file mode 100644 index 0000000..f0c2873 --- /dev/null +++ b/lib/tfw/components/history_monitor.py @@ -0,0 +1,37 @@ +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler +from collections import deque +from os.path import dirname + + +class CallbackEventHandler(FileSystemEventHandler): + def __init__(self, *callbacks): + super().__init__() + self.callbacks = callbacks + + def on_modified(self, event): + for callback in self.callbacks: + callback() + + +class HistoryMonitor: + def __init__(self, histfile): + self.histfile = histfile + self._history = deque() + self.observer = Observer() + self.observer.schedule(CallbackEventHandler(self._fetch_history), dirname(self.histfile)) + + @property + def history(self): + return self._history + + def _fetch_history(self): + with open(self.histfile, 'r') as ifile: + self._history = deque(ifile.readlines()) + + def watch(self): + self.observer.start() + + def stop(self): + self.observer.stop() + self.observer.join() From 5133806d33e4794b92b89ca6483f308bd15c6f5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 3 Mar 2018 22:44:05 +0100 Subject: [PATCH 09/28] Make Historymonitor.history a list instead of a deque (for slicing) --- lib/tfw/components/history_monitor.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index f0c2873..553d6bb 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -1,6 +1,5 @@ from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler -from collections import deque from os.path import dirname @@ -17,7 +16,7 @@ class CallbackEventHandler(FileSystemEventHandler): class HistoryMonitor: def __init__(self, histfile): self.histfile = histfile - self._history = deque() + self._history = [] self.observer = Observer() self.observer.schedule(CallbackEventHandler(self._fetch_history), dirname(self.histfile)) @@ -27,7 +26,7 @@ class HistoryMonitor: def _fetch_history(self): with open(self.histfile, 'r') as ifile: - self._history = deque(ifile.readlines()) + self._history = ifile.readlines() def watch(self): self.observer.start() From 180cf41f85657cd164e2967d78708d10ea43a7bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 3 Mar 2018 22:46:14 +0100 Subject: [PATCH 10/28] Implement history reading API --- Dockerfile | 4 +++- lib/tfw/components/terminado_event_handler.py | 13 +++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5772ee0..97e7759 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,7 +60,9 @@ ENV PYTHONPATH="/usr/local/lib/" \ TFW_LIB_DIR="/usr/local/lib/" \ TFW_CONTROLLER_DIR="/srv/controller" \ TFW_TERMINADO_DIR="/tmp/terminado_server" \ - TFW_FRONTEND_DIR="/srv/frontend" + TFW_FRONTEND_DIR="/srv/frontend" \ + TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history" \ + PROMPT_COMMAND="history -a" COPY nginx/nginx.conf ${TFW_NGINX_CONF} COPY nginx/components/ ${TFW_NGINX_COMPONENTS} diff --git a/lib/tfw/components/terminado_event_handler.py b/lib/tfw/components/terminado_event_handler.py index 00a90c9..9ae7dae 100644 --- a/lib/tfw/components/terminado_event_handler.py +++ b/lib/tfw/components/terminado_event_handler.py @@ -1,4 +1,5 @@ from tfw.components.terminado_mini_server import TerminadoMiniServer +from tfw.components.history_monitor import HistoryMonitor from tfw.event_handler_base import TriggerlessEventHandler from tfw.config import tfwenv from tfw.config.logs import logging @@ -9,14 +10,22 @@ class TerminadoEventHandler(TriggerlessEventHandler): def __init__(self, key): super().__init__(key) self.working_directory = tfwenv.TERMINADO_DIR + self.monitor = HistoryMonitor(tfwenv.HISTFILE) self.terminado_server = TerminadoMiniServer('/terminal', tfwenv.TERMINADO_PORT, tfwenv.TERMINADO_WD, ['bash']) - self.commands = {'write': self.write} + self.commands = {'write': self.write, + 'read': self.read} + self.monitor.watch() self.terminado_server.listen() def handle_event(self, key, data_json): log.debug('TerminadoEventHandler received event: {}'.format(data_json)) data = data_json['data'] - self.commands[data['command']](data) + data_json['data'] = self.commands[data['command']](data) + return data_json def write(self, data): self.terminado_server.pty.write(data['shellcmd']) + + def read(self, data): + data['history'] = self.monitor.history[-int(data.get('count', 1)):] + return data From cc964fdf27dbe792ca19adf798f9d278f92d5356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 3 Mar 2018 23:02:19 +0100 Subject: [PATCH 11/28] Strip newlines from stored history in HistoryMonitor --- lib/tfw/components/history_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index 553d6bb..4f0eba8 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -26,7 +26,7 @@ class HistoryMonitor: def _fetch_history(self): with open(self.histfile, 'r') as ifile: - self._history = ifile.readlines() + self._history = [line.rstrip() for line in ifile.readlines()] def watch(self): self.observer.start() From ab76e3d024d1765973d36fd6e8fdc607ae9ad020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sat, 3 Mar 2018 23:02:49 +0100 Subject: [PATCH 12/28] Store length of history list requested in every case --- lib/tfw/components/terminado_event_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/tfw/components/terminado_event_handler.py b/lib/tfw/components/terminado_event_handler.py index 9ae7dae..9368f34 100644 --- a/lib/tfw/components/terminado_event_handler.py +++ b/lib/tfw/components/terminado_event_handler.py @@ -27,5 +27,6 @@ class TerminadoEventHandler(TriggerlessEventHandler): self.terminado_server.pty.write(data['shellcmd']) def read(self, data): - data['history'] = self.monitor.history[-int(data.get('count', 1)):] + data['count'] = int(data.get('count', 1)) + data['history'] = self.monitor.history[-data['count']:] return data From 82073fda8cc2be943ee589d454076b28d90597d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sun, 4 Mar 2018 10:46:03 +0100 Subject: [PATCH 13/28] Rename TerminadoEventHandler's key to shell for consistency --- src/demo/event_handler_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/demo/event_handler_main.py b/src/demo/event_handler_main.py index b7090d3..890bee0 100644 --- a/src/demo/event_handler_main.py +++ b/src/demo/event_handler_main.py @@ -8,7 +8,7 @@ from tfw.config import tfwenv if __name__ == '__main__': ide = SourceCodeEventHandler('webide', tfwenv.WEBIDE_WD) - terminado = TerminadoEventHandler('terminado') + terminado = TerminadoEventHandler('shell') processmanager = ProcessManagingEventHandler('processmanager', ide.monitor) eventhandlers = {ide, terminado, processmanager} From ac8e5506a5a96c25d20cd6f0474ba4b76376f3ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Sun, 4 Mar 2018 11:28:34 +0100 Subject: [PATCH 14/28] Improve handling of bash history --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 97e7759..3e451e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,7 +62,12 @@ ENV PYTHONPATH="/usr/local/lib/" \ TFW_TERMINADO_DIR="/tmp/terminado_server" \ TFW_FRONTEND_DIR="/srv/frontend" \ TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history" \ - PROMPT_COMMAND="history -a" + PROMPT_COMMAND="history -a" \ + HISTFILESIZE=1000 \ + HISTSIZE=1000 + +RUN sudo -u ${AVATAO_USER} bash -c 'shopt -s cmdhist' &&\ + sudo -u ${AVATAO_USER} bash -c 'shopt -s histappend' COPY nginx/nginx.conf ${TFW_NGINX_CONF} COPY nginx/components/ ${TFW_NGINX_COMPONENTS} From 100ec96e6372cfafb2e5176b90d8e49cbf27a972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 5 Mar 2018 13:30:34 +0100 Subject: [PATCH 15/28] Fix shopt magic in Dockerfile --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3e451e5..fd700c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,8 +66,7 @@ ENV PYTHONPATH="/usr/local/lib/" \ HISTFILESIZE=1000 \ HISTSIZE=1000 -RUN sudo -u ${AVATAO_USER} bash -c 'shopt -s cmdhist' &&\ - sudo -u ${AVATAO_USER} bash -c 'shopt -s histappend' +RUN echo "shopt -s cmdhist && shopt -s histappend" >> .bashrc COPY nginx/nginx.conf ${TFW_NGINX_CONF} COPY nginx/components/ ${TFW_NGINX_COMPONENTS} From 7dd25123005d4e86b7da1f672eafd6993607c995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 5 Mar 2018 16:17:56 +0100 Subject: [PATCH 16/28] Fix history monitoring stuff in Dockerfile (AVATAO baseimage bashrc) --- Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index fd700c2..00edd03 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,11 +62,15 @@ ENV PYTHONPATH="/usr/local/lib/" \ TFW_TERMINADO_DIR="/tmp/terminado_server" \ TFW_FRONTEND_DIR="/srv/frontend" \ TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history" \ - PROMPT_COMMAND="history -a" \ - HISTFILESIZE=1000 \ - HISTSIZE=1000 + PROMPT_COMMAND="history -a" -RUN echo "shopt -s cmdhist && shopt -s histappend" >> .bashrc +RUN echo "shopt -s cmdhist\n" \ + "shopt -s histappend\n" \ + "unset HISTCONTROL\n" \ + "export HISTFILESIZE=1000\n" \ + "export HISTSIZE=1000\n" \ + 'PROMPT_COMMAND="history -a"\n' \ + >> /home/${AVATAO_USER}/.bashrc COPY nginx/nginx.conf ${TFW_NGINX_CONF} COPY nginx/components/ ${TFW_NGINX_COMPONENTS} From 0a20cffa0949312d85ec9e696d601427bb4c48c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 5 Mar 2018 16:23:01 +0100 Subject: [PATCH 17/28] Implement callback subscription logic in HistoryMonitor --- lib/tfw/components/history_monitor.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index 4f0eba8..395149d 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -17,17 +17,31 @@ class HistoryMonitor: def __init__(self, histfile): self.histfile = histfile self._history = [] + self._last_length = len(self._history) + self._callbacks = [] self.observer = Observer() - self.observer.schedule(CallbackEventHandler(self._fetch_history), dirname(self.histfile)) + self.observer.schedule(CallbackEventHandler(self._fetch_history, + self._invoke_callbacks), + dirname(self.histfile)) @property def history(self): return self._history + @property + def callbacks(self): + return self._callbacks + def _fetch_history(self): + self._last_length = len(self._history) with open(self.histfile, 'r') as ifile: self._history = [line.rstrip() for line in ifile.readlines()] + def _invoke_callbacks(self): + if self._last_length < len(self._history): + for callback in self.callbacks: + callback(self.history) + def watch(self): self.observer.start() From a4b07e549e190b77cf7404d4ca971fd2492646c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 5 Mar 2018 16:24:36 +0100 Subject: [PATCH 18/28] Rename & expose as property TerminadoEventHandler.monitor --- lib/tfw/components/terminado_event_handler.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/tfw/components/terminado_event_handler.py b/lib/tfw/components/terminado_event_handler.py index 9368f34..e6fa2cd 100644 --- a/lib/tfw/components/terminado_event_handler.py +++ b/lib/tfw/components/terminado_event_handler.py @@ -10,13 +10,17 @@ class TerminadoEventHandler(TriggerlessEventHandler): def __init__(self, key): super().__init__(key) self.working_directory = tfwenv.TERMINADO_DIR - self.monitor = HistoryMonitor(tfwenv.HISTFILE) + self._historymonitor = HistoryMonitor(tfwenv.HISTFILE) self.terminado_server = TerminadoMiniServer('/terminal', tfwenv.TERMINADO_PORT, tfwenv.TERMINADO_WD, ['bash']) self.commands = {'write': self.write, 'read': self.read} - self.monitor.watch() + self._historymonitor.watch() self.terminado_server.listen() + @property + def historymonitor(self): + return self._historymonitor + def handle_event(self, key, data_json): log.debug('TerminadoEventHandler received event: {}'.format(data_json)) data = data_json['data'] @@ -28,5 +32,5 @@ class TerminadoEventHandler(TriggerlessEventHandler): def read(self, data): data['count'] = int(data.get('count', 1)) - data['history'] = self.monitor.history[-data['count']:] + data['history'] = self.historymonitor.history[-data['count']:] return data From d88728bb711e5c3081852f1f4cec1e142df16f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 5 Mar 2018 16:36:59 +0100 Subject: [PATCH 19/28] Implement command logging as exampe for HistoryMonitor usage --- src/demo/event_handler_main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/demo/event_handler_main.py b/src/demo/event_handler_main.py index 890bee0..8500d09 100644 --- a/src/demo/event_handler_main.py +++ b/src/demo/event_handler_main.py @@ -1,14 +1,17 @@ +from tornado.ioloop import IOLoop + from tfw.components.source_code_event_handler import SourceCodeEventHandler from tfw.components.terminado_event_handler import TerminadoEventHandler from tfw.components.process_managing_event_handler import ProcessManagingEventHandler -from tornado.ioloop import IOLoop - from tfw.config import tfwenv +from tfw.config.logs import logging +log = logging.getLogger(__name__) if __name__ == '__main__': ide = SourceCodeEventHandler('webide', tfwenv.WEBIDE_WD) terminado = TerminadoEventHandler('shell') + terminado.historymonitor.callbacks.append(lambda hist: log.debug('User executed command: "{}"'.format(hist[-1]))) processmanager = ProcessManagingEventHandler('processmanager', ide.monitor) eventhandlers = {ide, terminado, processmanager} From 02bc31700958bf55c5d7bd59207c7b34b8411ff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 5 Mar 2018 16:57:50 +0100 Subject: [PATCH 20/28] Refactor CallbackEventHandler to be a PatternMatchingEventHandler --- lib/tfw/components/history_monitor.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/tfw/components/history_monitor.py b/lib/tfw/components/history_monitor.py index 395149d..f07a57b 100644 --- a/lib/tfw/components/history_monitor.py +++ b/lib/tfw/components/history_monitor.py @@ -1,11 +1,11 @@ from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler +from watchdog.events import PatternMatchingEventHandler from os.path import dirname -class CallbackEventHandler(FileSystemEventHandler): - def __init__(self, *callbacks): - super().__init__() +class CallbackEventHandler(PatternMatchingEventHandler): + def __init__(self, files, *callbacks): + super().__init__(files) self.callbacks = callbacks def on_modified(self, event): @@ -20,7 +20,8 @@ class HistoryMonitor: self._last_length = len(self._history) self._callbacks = [] self.observer = Observer() - self.observer.schedule(CallbackEventHandler(self._fetch_history, + self.observer.schedule(CallbackEventHandler([self.histfile], + self._fetch_history, self._invoke_callbacks), dirname(self.histfile)) From 021a1225234d5bf44b7cc8856ff2874e6c19815d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 6 Mar 2018 13:16:52 +0100 Subject: [PATCH 21/28] Stop HistoryMonitor in TerminadoEventHandler.cleanup() --- lib/tfw/components/terminado_event_handler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/tfw/components/terminado_event_handler.py b/lib/tfw/components/terminado_event_handler.py index e6fa2cd..756d4db 100644 --- a/lib/tfw/components/terminado_event_handler.py +++ b/lib/tfw/components/terminado_event_handler.py @@ -34,3 +34,6 @@ class TerminadoEventHandler(TriggerlessEventHandler): data['count'] = int(data.get('count', 1)) data['history'] = self.historymonitor.history[-data['count']:] return data + + def cleanup(self): + self.historymonitor.stop() From 555d5711123510245f018b9e87417cbeb240ef48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 6 Mar 2018 14:28:36 +0100 Subject: [PATCH 22/28] Expose term_manager in TerminadoMiniServer as a property --- lib/tfw/components/terminado_mini_server.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/tfw/components/terminado_mini_server.py b/lib/tfw/components/terminado_mini_server.py index 26afe31..691099a 100644 --- a/lib/tfw/components/terminado_mini_server.py +++ b/lib/tfw/components/terminado_mini_server.py @@ -10,16 +10,20 @@ log = logging.getLogger(__name__) class TerminadoMiniServer: def __init__(self, url, port, workdir, shellcmd): self.port = port - self.term_manager = SingleTermManager(shell_command=shellcmd, - term_settings={'cwd': workdir}) + self._term_manager = SingleTermManager(shell_command=shellcmd, + term_settings={'cwd': workdir}) self.application = Application( [( url, TerminadoMiniServer.CORSTermSocket, - {'term_manager': self.term_manager} + {'term_manager': self._term_manager} )] ) + @property + def term_manager(self): + return self._term_manager + @property def pty(self): return self.term_manager.terminal.ptyproc From e20a5dab23061db98075798a5ea75f052dee9278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 6 Mar 2018 14:33:01 +0100 Subject: [PATCH 23/28] Implement pty recovery --- lib/tfw/components/terminado_event_handler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/tfw/components/terminado_event_handler.py b/lib/tfw/components/terminado_event_handler.py index 756d4db..0eb423d 100644 --- a/lib/tfw/components/terminado_event_handler.py +++ b/lib/tfw/components/terminado_event_handler.py @@ -13,7 +13,8 @@ class TerminadoEventHandler(TriggerlessEventHandler): self._historymonitor = HistoryMonitor(tfwenv.HISTFILE) self.terminado_server = TerminadoMiniServer('/terminal', tfwenv.TERMINADO_PORT, tfwenv.TERMINADO_WD, ['bash']) self.commands = {'write': self.write, - 'read': self.read} + 'read': self.read, + 'resetshell': self.resetshell} self._historymonitor.watch() self.terminado_server.listen() @@ -35,5 +36,9 @@ class TerminadoEventHandler(TriggerlessEventHandler): data['history'] = self.historymonitor.history[-data['count']:] return data + def resetshell(self, data): + self.terminado_server.term_manager.terminal = None + self.terminado_server.term_manager.get_terminal() + def cleanup(self): self.historymonitor.stop() From 1165fa4ec941bfc0ecedfb2c875663bbaa1aa0c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 6 Mar 2018 16:15:04 +0100 Subject: [PATCH 24/28] Refactor pty resetting logic to TerminadoMiniServer --- lib/tfw/components/terminado_event_handler.py | 7 +------ lib/tfw/components/terminado_mini_server.py | 8 ++++++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/tfw/components/terminado_event_handler.py b/lib/tfw/components/terminado_event_handler.py index 0eb423d..756d4db 100644 --- a/lib/tfw/components/terminado_event_handler.py +++ b/lib/tfw/components/terminado_event_handler.py @@ -13,8 +13,7 @@ class TerminadoEventHandler(TriggerlessEventHandler): self._historymonitor = HistoryMonitor(tfwenv.HISTFILE) self.terminado_server = TerminadoMiniServer('/terminal', tfwenv.TERMINADO_PORT, tfwenv.TERMINADO_WD, ['bash']) self.commands = {'write': self.write, - 'read': self.read, - 'resetshell': self.resetshell} + 'read': self.read} self._historymonitor.watch() self.terminado_server.listen() @@ -36,9 +35,5 @@ class TerminadoEventHandler(TriggerlessEventHandler): data['history'] = self.historymonitor.history[-data['count']:] return data - def resetshell(self, data): - self.terminado_server.term_manager.terminal = None - self.terminado_server.term_manager.get_terminal() - def cleanup(self): self.historymonitor.stop() diff --git a/lib/tfw/components/terminado_mini_server.py b/lib/tfw/components/terminado_mini_server.py index 691099a..153a382 100644 --- a/lib/tfw/components/terminado_mini_server.py +++ b/lib/tfw/components/terminado_mini_server.py @@ -15,7 +15,7 @@ class TerminadoMiniServer: self.application = Application( [( url, - TerminadoMiniServer.CORSTermSocket, + TerminadoMiniServer.ResetterTermSocket, {'term_manager': self._term_manager} )] ) @@ -28,10 +28,14 @@ class TerminadoMiniServer: def pty(self): return self.term_manager.terminal.ptyproc - class CORSTermSocket(TermSocket): + class ResetterTermSocket(TermSocket): def check_origin(self, origin): return True + def on_close(self): + self.term_manager.terminal = None + self.term_manager.get_terminal() + def listen(self): self.application.listen(self.port) From 3cf404ee9ad9e7b093913752ce488dc6837894aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 7 Mar 2018 09:17:29 +0100 Subject: [PATCH 25/28] Implement inotify event ignoration logic --- lib/tfw/components/directory_monitor.py | 13 +++++++++++++ lib/tfw/components/source_code_event_handler.py | 6 +++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index 6d2d264..fd69cbb 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -13,6 +13,7 @@ class WebideReloadEventHandler(FileSystemEventHandler): super().__init__() self.uplink = ServerUplinkConnector() self._paused = False + self.ignore = 0 def pause(self): self._paused = True @@ -23,6 +24,9 @@ class WebideReloadEventHandler(FileSystemEventHandler): @RateLimiter(rate_per_second=5) def on_modified(self, event): if self._paused: return + if self.ignore > 0: + self.ignore = self.ignore - 1 + return log.debug(event) key = 'webide' self.uplink.send(key, {'data': {'command': 'reload'}}) @@ -42,6 +46,15 @@ class DirectoryMonitor: self.observer.stop() self.observer.join() + @property + def ignore(self): + return self.eventhandler.ignore + + @ignore.setter + def ignore(self, value): + if value >= 0: self.ignore = value + else: self.ignore = 0 + @property def pauser(self): return DirectoryMonitor.Pauser(self) diff --git a/lib/tfw/components/source_code_event_handler.py b/lib/tfw/components/source_code_event_handler.py index 19896d4..dd26072 100644 --- a/lib/tfw/components/source_code_event_handler.py +++ b/lib/tfw/components/source_code_event_handler.py @@ -68,9 +68,9 @@ class SourceCodeEventHandler(TriggerlessEventHandler): return data def write(self, data): - with self.monitor.pauser: - try: self.filemanager.file_contents = data['content'] - except Exception: log.exception('Error writing file!') + self.monitor.eventhandler.ignore = 1 + try: self.filemanager.file_contents = data['content'] + except Exception: log.exception('Error writing file!') del data['content'] return data From e064c34d8f46a91fdb76c2207671c3af68646008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 7 Mar 2018 11:34:21 +0100 Subject: [PATCH 26/28] Enforce HISTFILE consistency --- Dockerfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 00edd03..cd0ce9f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,12 +64,13 @@ ENV PYTHONPATH="/usr/local/lib/" \ TFW_HISTFILE="/home/${AVATAO_USER}/.bash_history" \ PROMPT_COMMAND="history -a" -RUN echo "shopt -s cmdhist\n" \ - "shopt -s histappend\n" \ - "unset HISTCONTROL\n" \ - "export HISTFILESIZE=1000\n" \ - "export HISTSIZE=1000\n" \ - 'PROMPT_COMMAND="history -a"\n' \ +RUN echo "shopt -s cmdhist\n" \ + "shopt -s histappend\n" \ + "unset HISTCONTROL\n" \ + "export HISTFILE=$TFW_HISTFILE\n" \ + "export HISTFILESIZE=1000\n" \ + "export HISTSIZE=1000\n" \ + 'PROMPT_COMMAND="history -a"\n' \ >> /home/${AVATAO_USER}/.bashrc COPY nginx/nginx.conf ${TFW_NGINX_CONF} From cb7e8c5663860560bc03b27e65e6378ac55eebc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 7 Mar 2018 11:36:21 +0100 Subject: [PATCH 27/28] Make DirectoryMonitor.ignore setter more pythonic --- lib/tfw/components/directory_monitor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index fd69cbb..1493133 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -52,8 +52,7 @@ class DirectoryMonitor: @ignore.setter def ignore(self, value): - if value >= 0: self.ignore = value - else: self.ignore = 0 + self.ignore = value if value >= 0 else 0 @property def pauser(self): From 6d59762e2b0f94e3bcbeec1be486a860780b174a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Wed, 7 Mar 2018 11:37:52 +0100 Subject: [PATCH 28/28] Clean up confusing usage of data_json['data'] in TerminadoEventHandler --- lib/tfw/components/terminado_event_handler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/tfw/components/terminado_event_handler.py b/lib/tfw/components/terminado_event_handler.py index 756d4db..4a8f98a 100644 --- a/lib/tfw/components/terminado_event_handler.py +++ b/lib/tfw/components/terminado_event_handler.py @@ -23,8 +23,7 @@ class TerminadoEventHandler(TriggerlessEventHandler): def handle_event(self, key, data_json): log.debug('TerminadoEventHandler received event: {}'.format(data_json)) - data = data_json['data'] - data_json['data'] = self.commands[data['command']](data) + data_json['data'] = self.commands[data_json['data']['command']](data_json['data']) return data_json def write(self, data):