From 9c006451bf96363a60cf2e76f42391da75d60901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 12 Feb 2018 16:01:24 +0100 Subject: [PATCH 1/9] Implement proof of concept directory event monitoring --- lib/tfw/components/directory_monitor.py | 26 +++++++++++++++++++ .../components/source_code_event_handler.py | 4 +++ requirements.txt | 1 + 3 files changed, 31 insertions(+) create mode 100644 lib/tfw/components/directory_monitor.py diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py new file mode 100644 index 0000000..18ddc4c --- /dev/null +++ b/lib/tfw/components/directory_monitor.py @@ -0,0 +1,26 @@ +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +from tfw.message_sender import MessageSender + + +class MessagingEventHandler(FileSystemEventHandler): + def __init__(self): + super().__init__() + self.message_sender = MessageSender() + + def on_modified(self, event): + self.message_sender.send('Watchdog', 'CECA') + + +class DirectoryMonitor: + def __init__(self, directory): + self.observer = Observer() + self.observer.schedule(MessagingEventHandler(), directory, recursive=True) + + def watch(self): + self.observer.start() + + def stop(self): + self.observer.stop() + self.observer.join() diff --git a/lib/tfw/components/source_code_event_handler.py b/lib/tfw/components/source_code_event_handler.py index abb5738..eb22dce 100644 --- a/lib/tfw/components/source_code_event_handler.py +++ b/lib/tfw/components/source_code_event_handler.py @@ -3,6 +3,7 @@ from glob import glob from tfw.util import SupervisorMixin from tfw.event_handler_base import EventHandlerBase +from tfw.components.directory_monitor import DirectoryMonitor from tfw.config.logs import logging log = logging.getLogger(__name__) @@ -56,6 +57,9 @@ class SourceCodeEventHandler(EventHandlerBase, SupervisorMixin): 'select': self.select } + self.monitor = DirectoryMonitor(directory) + self.monitor.watch() # This runs on a separate thread TODO: when to call stop()? + def read(self, data): try: data['content'] = self.filemanager.file_contents except PermissionError: data['content'] = 'You have no permission to open that file :(' diff --git a/requirements.txt b/requirements.txt index 5cfef8d..e057890 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ tornado==4.5.3 pyzmq==16.0.4 transitions==0.6.4 terminado==0.8.1 +watchdog==0.8.3 From 67579aea29f15100bf3c33ac66c3b05f29edf835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 12 Feb 2018 16:43:30 +0100 Subject: [PATCH 2/9] Implement sending reload command on file system changes --- lib/tfw/components/directory_monitor.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index 18ddc4c..ffd8708 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -1,16 +1,22 @@ from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler -from tfw.message_sender import MessageSender +from tfw.networking.event_handlers.server_connector import ServerUplinkConnector + +from tfw.config.logs import logging +log = logging.getLogger(__name__) class MessagingEventHandler(FileSystemEventHandler): def __init__(self): super().__init__() - self.message_sender = MessageSender() + self.uplink = ServerUplinkConnector() def on_modified(self, event): - self.message_sender.send('Watchdog', 'CECA') + log.debug(event) + anchor = 'anchor_webide' + self.uplink.send(anchor, {'anchor': anchor, + 'data': {'command': 'reload'}}) class DirectoryMonitor: From 955e1c1cf330228d26413dddea8806837f9573d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 12 Feb 2018 17:01:23 +0100 Subject: [PATCH 3/9] Rename watchdog FileSystemEventHandler child --- lib/tfw/components/directory_monitor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index ffd8708..6531185 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -7,7 +7,7 @@ from tfw.config.logs import logging log = logging.getLogger(__name__) -class MessagingEventHandler(FileSystemEventHandler): +class WebideReloadEventHandler(FileSystemEventHandler): def __init__(self): super().__init__() self.uplink = ServerUplinkConnector() @@ -22,7 +22,7 @@ class MessagingEventHandler(FileSystemEventHandler): class DirectoryMonitor: def __init__(self, directory): self.observer = Observer() - self.observer.schedule(MessagingEventHandler(), directory, recursive=True) + self.observer.schedule(WebideReloadEventHandler(), directory, recursive=True) def watch(self): self.observer.start() From dceb4b2b7ee2441c8e729d522ad388f958e4a09a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Mon, 12 Feb 2018 17:46:01 +0100 Subject: [PATCH 4/9] Implement basic rate limiting for webide autoreload magic --- lib/tfw/components/directory_monitor.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index 6531185..627fb31 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -1,3 +1,4 @@ +from time import time from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler @@ -8,21 +9,32 @@ log = logging.getLogger(__name__) class WebideReloadEventHandler(FileSystemEventHandler): - def __init__(self): + def __init__(self, rate_per_second): super().__init__() self.uplink = ServerUplinkConnector() + self.min_interval = 1 / float(rate_per_second) + self.last_call = time() + def on_modified(self, event): + if self.limit_rate(): return + log.debug(event) anchor = 'anchor_webide' self.uplink.send(anchor, {'anchor': anchor, 'data': {'command': 'reload'}}) + def limit_rate(self): #TODO: pls review me :3 + since_last_call = time() - self.last_call + to_next_call = self.min_interval - since_last_call + self.last_call = time() + return to_next_call > 0 + class DirectoryMonitor: def __init__(self, directory): self.observer = Observer() - self.observer.schedule(WebideReloadEventHandler(), directory, recursive=True) + self.observer.schedule(WebideReloadEventHandler(rate_per_second=5), directory, recursive=True) def watch(self): self.observer.start() From 4d49c8d11b47292e390ff78d0f997a8f3fdadcf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 13 Feb 2018 14:37:56 +0100 Subject: [PATCH 5/9] Improve webide refresh trigger rate limiting --- lib/tfw/components/directory_monitor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index 627fb31..9feb46d 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -1,4 +1,4 @@ -from time import time +from time import time, sleep from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler @@ -17,7 +17,7 @@ class WebideReloadEventHandler(FileSystemEventHandler): self.last_call = time() def on_modified(self, event): - if self.limit_rate(): return + self.limit_rate() log.debug(event) anchor = 'anchor_webide' @@ -28,7 +28,8 @@ class WebideReloadEventHandler(FileSystemEventHandler): since_last_call = time() - self.last_call to_next_call = self.min_interval - since_last_call self.last_call = time() - return to_next_call > 0 + if to_next_call > 0: + sleep(to_next_call) class DirectoryMonitor: From 60bcb8c2b052ec86b82b4153399ebd5fa626ce75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 13 Feb 2018 14:55:33 +0100 Subject: [PATCH 6/9] Refactor webide rate limiting --- lib/tfw/components/directory_monitor.py | 36 ++++++++++++++++--------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index 9feb46d..5db6d6f 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -1,6 +1,7 @@ from time import time, sleep from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler +from functools import wraps from tfw.networking.event_handlers.server_connector import ServerUplinkConnector @@ -8,23 +9,19 @@ from tfw.config.logs import logging log = logging.getLogger(__name__) -class WebideReloadEventHandler(FileSystemEventHandler): +class RateLimiter: def __init__(self, rate_per_second): - super().__init__() - self.uplink = ServerUplinkConnector() - self.min_interval = 1 / float(rate_per_second) self.last_call = time() - def on_modified(self, event): - self.limit_rate() + def __call__(self, fun): + @wraps(fun) + def wrapper(*args, **kwargs): + self._limit_rate() + fun(*args, **kwargs) + return wrapper - log.debug(event) - anchor = 'anchor_webide' - self.uplink.send(anchor, {'anchor': anchor, - 'data': {'command': 'reload'}}) - - def limit_rate(self): #TODO: pls review me :3 + def _limit_rate(self): #TODO: pls review me :3 since_last_call = time() - self.last_call to_next_call = self.min_interval - since_last_call self.last_call = time() @@ -32,10 +29,23 @@ class WebideReloadEventHandler(FileSystemEventHandler): sleep(to_next_call) +class WebideReloadEventHandler(FileSystemEventHandler): + def __init__(self): + super().__init__() + self.uplink = ServerUplinkConnector() + + @RateLimiter(rate_per_second=5) + def on_modified(self, event): + log.debug(event) + anchor = 'anchor_webide' + self.uplink.send(anchor, {'anchor': anchor, + 'data': {'command': 'reload'}}) + + class DirectoryMonitor: def __init__(self, directory): self.observer = Observer() - self.observer.schedule(WebideReloadEventHandler(rate_per_second=5), directory, recursive=True) + self.observer.schedule(WebideReloadEventHandler(), directory, recursive=True) def watch(self): self.observer.start() From fd029dbfe707ace324b9da8db87e30606b884228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 13 Feb 2018 15:02:48 +0100 Subject: [PATCH 7/9] Move RateLimiter to tfw.util --- lib/tfw/components/directory_monitor.py | 22 +--------------------- lib/tfw/util.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/lib/tfw/components/directory_monitor.py b/lib/tfw/components/directory_monitor.py index 5db6d6f..5dae10e 100644 --- a/lib/tfw/components/directory_monitor.py +++ b/lib/tfw/components/directory_monitor.py @@ -1,33 +1,13 @@ -from time import time, sleep from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler -from functools import wraps from tfw.networking.event_handlers.server_connector import ServerUplinkConnector +from tfw.util import RateLimiter from tfw.config.logs import logging log = logging.getLogger(__name__) -class RateLimiter: - def __init__(self, rate_per_second): - self.min_interval = 1 / float(rate_per_second) - self.last_call = time() - - def __call__(self, fun): - @wraps(fun) - def wrapper(*args, **kwargs): - self._limit_rate() - fun(*args, **kwargs) - return wrapper - - def _limit_rate(self): #TODO: pls review me :3 - since_last_call = time() - self.last_call - to_next_call = self.min_interval - since_last_call - self.last_call = time() - if to_next_call > 0: - sleep(to_next_call) - class WebideReloadEventHandler(FileSystemEventHandler): def __init__(self): diff --git a/lib/tfw/util.py b/lib/tfw/util.py index 4194953..ab9ab83 100644 --- a/lib/tfw/util.py +++ b/lib/tfw/util.py @@ -1,6 +1,8 @@ import xmlrpc.client, zmq from contextlib import suppress from xmlrpc.client import Fault as SupervisorFault +from time import time, sleep +from functools import wraps from tfw.config import tfwenv @@ -31,3 +33,23 @@ class SupervisorMixin: class ZMQConnectorBase: def __init__(self, zmq_context=None): self._zmq_context = zmq_context or zmq.Context.instance() + + +class RateLimiter: + def __init__(self, rate_per_second): + self.min_interval = 1 / float(rate_per_second) + self.last_call = time() + + def __call__(self, fun): + @wraps(fun) + def wrapper(*args, **kwargs): + self._limit_rate() + fun(*args, **kwargs) + return wrapper + + def _limit_rate(self): #TODO: pls review me :3 + since_last_call = time() - self.last_call + to_next_call = self.min_interval - since_last_call + self.last_call = time() + if to_next_call > 0: + sleep(to_next_call) \ No newline at end of file From 1d47ca5684db5addde084ca0a5b54c4cda0bdcb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 13 Feb 2018 15:38:46 +0100 Subject: [PATCH 8/9] Add method EventHandlerBase.cleanup() --- lib/tfw/components/source_code_event_handler.py | 5 ++++- lib/tfw/event_handler_base.py | 3 +++ src/demo/event_handler_main.py | 9 ++++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/tfw/components/source_code_event_handler.py b/lib/tfw/components/source_code_event_handler.py index eb22dce..5b5de40 100644 --- a/lib/tfw/components/source_code_event_handler.py +++ b/lib/tfw/components/source_code_event_handler.py @@ -58,7 +58,7 @@ class SourceCodeEventHandler(EventHandlerBase, SupervisorMixin): } self.monitor = DirectoryMonitor(directory) - self.monitor.watch() # This runs on a separate thread TODO: when to call stop()? + self.monitor.watch() # This runs on a separate thread def read(self, data): try: data['content'] = self.filemanager.file_contents @@ -89,6 +89,9 @@ class SourceCodeEventHandler(EventHandlerBase, SupervisorMixin): self.attach_fileinfo(data) return data_json + def cleanup(self): + self.monitor.stop() + def map_file_extension_to_language(filename): language_map = { diff --git a/lib/tfw/event_handler_base.py b/lib/tfw/event_handler_base.py index 4dfdb84..2841e0f 100644 --- a/lib/tfw/event_handler_base.py +++ b/lib/tfw/event_handler_base.py @@ -23,6 +23,9 @@ class EventHandlerBase: def handle_reset(self, data_json): return None + def cleanup(self): + pass + def message_other(self, anchor, data): message = { 'anchor': anchor, diff --git a/src/demo/event_handler_main.py b/src/demo/event_handler_main.py index f7623a9..dda5c36 100644 --- a/src/demo/event_handler_main.py +++ b/src/demo/event_handler_main.py @@ -6,6 +6,9 @@ from tfw.config import tfwenv if __name__ == '__main__': - anchor_webide = SourceCodeEventHandler('anchor_webide', tfwenv.WEBIDE_WD, 'login') - anchor_terminado = TerminadoEventHandler('anchor_terminado', 'terminado') - IOLoop.instance().start() + eventhandlers = {SourceCodeEventHandler('anchor_webide', tfwenv.WEBIDE_WD, 'login'), + TerminadoEventHandler('anchor_terminado', 'terminado')} + try: + IOLoop.instance().start() + finally: + for eh in eventhandlers: eh.cleanup() From 5213152477e9600536c3d2d08898d32b385aea76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20T=C3=B3th?= Date: Tue, 13 Feb 2018 16:29:45 +0100 Subject: [PATCH 9/9] Remove obsolete TODO --- lib/tfw/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tfw/util.py b/lib/tfw/util.py index ab9ab83..7ee99cc 100644 --- a/lib/tfw/util.py +++ b/lib/tfw/util.py @@ -47,7 +47,7 @@ class RateLimiter: fun(*args, **kwargs) return wrapper - def _limit_rate(self): #TODO: pls review me :3 + def _limit_rate(self): since_last_call = time() - self.last_call to_next_call = self.min_interval - since_last_call self.last_call = time()