Simplify package structure

tfw/components/ide/__init__.py

@@ -0,0 +1 @@
+from .ide_handler import IdeHandler

tfw/components/ide/file_manager/__init__.py

@@ -0,0 +1 @@
+from .file_manager import FileManager

tfw/components/ide/file_manager/file_manager.py

@@ -0,0 +1,93 @@
+from typing import Iterable
+from glob import glob
+from fnmatch import fnmatchcase
+from os.path import basename, isfile, join, relpath, exists, isdir, realpath
+
+
+class FileManager:  # pylint: disable=too-many-instance-attributes
+    def __init__(self, working_directory, allowed_directories, selected_file=None, exclude=None):
+        self._exclude, self.exclude = [], exclude
+        self._allowed_directories, self.allowed_directories = None, allowed_directories
+        self._workdir, self.workdir = None, working_directory
+        self._filename, self.filename = None, selected_file or self.files[0]
+
+    @property
+    def exclude(self):
+        return self._exclude
+
+    @exclude.setter
+    def exclude(self, exclude):
+        if exclude is None:
+            return
+        if not isinstance(exclude, Iterable):
+            raise TypeError('Exclude must be Iterable!')
+        self._exclude = exclude
+
+    @property
+    def workdir(self):
+        return self._workdir
+
+    @workdir.setter
+    def workdir(self, directory):
+        if not exists(directory) or not isdir(directory):
+            raise EnvironmentError(f'"{directory}" is not a directory!')
+        if not self._is_in_allowed_dir(directory):
+            raise EnvironmentError(f'Directory "{directory}" is not allowed!')
+        self._workdir = directory
+
+    @property
+    def allowed_directories(self):
+        return self._allowed_directories
+
+    @allowed_directories.setter
+    def allowed_directories(self, directories):
+        self._allowed_directories = [realpath(directory) for directory in directories]
+
+    @property
+    def filename(self):
+        return self._filename
+
+    @filename.setter
+    def filename(self, filename):
+        if filename not in self.files:
+            raise EnvironmentError('No such file in workdir!')
+        self._filename = filename
+
+    @property
+    def files(self):
+        return [
+            self._relpath(file)
+            for file in glob(join(self._workdir, '**/*'), recursive=True)
+            if isfile(file)
+            and self._is_in_allowed_dir(file)
+            and not self._is_blacklisted(file)
+        ]
+
+    @property
+    def file_contents(self):
+        with open(self._filepath(self.filename), 'rb', buffering=0) as ifile:
+            return ifile.read().decode(errors='surrogateescape')
+
+    @file_contents.setter
+    def file_contents(self, value):
+        with open(self._filepath(self.filename), 'wb', buffering=0) as ofile:
+            ofile.write(value.encode())
+
+    def _is_in_allowed_dir(self, path):
+        return any(
+            realpath(path).startswith(allowed_dir)
+            for allowed_dir in self.allowed_directories
+        )
+
+    def _is_blacklisted(self, file):
+        return any(
+            fnmatchcase(file, blacklisted) or
+            fnmatchcase(basename(file), blacklisted)
+            for blacklisted in self.exclude
+        )
+
+    def _filepath(self, filename):
+        return join(self._workdir, filename)
+
+    def _relpath(self, filename):
+        return relpath(self._filepath(filename), start=self._workdir)

tfw/components/ide/file_manager/test_file_manager.py

@@ -0,0 +1,124 @@
+# pylint: disable=redefined-outer-name
+
+from dataclasses import dataclass
+from secrets import token_urlsafe
+from os.path import join
+from os import chdir, mkdir, symlink
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import pytest
+
+from .file_manager import FileManager
+
+@dataclass
+class ManagerContext:
+    folder: str
+    manager: FileManager
+
+    def join(self, path):
+        return join(self.folder, path)
+
+
+@pytest.fixture()
+def context():
+    dirs = {}
+
+    with TemporaryDirectory() as workdir:
+        chdir(workdir)
+        for name in ['allowed', 'excluded', 'invis']:
+            node = join(workdir, name)
+            mkdir(node)
+            Path(join(node, 'empty.txt')).touch()
+            Path(join(node, 'empty.bin')).touch()
+            dirs[name] = node
+
+        yield ManagerContext(
+            workdir,
+            FileManager(
+                dirs['allowed'],
+                [dirs['allowed'], dirs['excluded']],
+                exclude=['*/excluded/*']
+            )
+        )
+
+@pytest.mark.parametrize('subdir', ['allowed/', 'excluded/'])
+def test_select_allowed_dirs(context, subdir):
+    context.manager.workdir = context.join(subdir)
+    assert context.manager.workdir == context.join(subdir)
+    newdir = context.join(subdir+'deep')
+    mkdir(newdir)
+    context.manager.workdir = newdir
+    assert context.manager.workdir == newdir
+
+@pytest.mark.parametrize('invdir', ['', 'invis'])
+def test_select_forbidden_dirs(context, invdir):
+    fullpath = context.join(invdir)
+    with pytest.raises(OSError):
+        context.manager.workdir = fullpath
+    assert context.manager.workdir != fullpath
+    context.manager.allowed_directories += [fullpath]
+    context.manager.workdir = fullpath
+    assert context.manager.workdir == fullpath
+
+
+@pytest.mark.parametrize('filename', ['another.txt', '*.txt'])
+def test_select_allowed_files(context, filename):
+    Path(context.join('allowed/'+filename)).touch()
+    assert filename in context.manager.files
+    context.manager.filename = filename
+    assert context.manager.filename == filename
+
+@pytest.mark.parametrize('path', [
+    {'dir': 'allowed/', 'file': 'illegal.bin'},
+    {'dir': 'excluded/', 'file': 'legal.txt'},
+    {'dir': 'allowed/', 'file': token_urlsafe(16)+'.bin'},
+    {'dir': 'excluded/', 'file': token_urlsafe(16)+'.txt'},
+    {'dir': 'allowed/', 'file': token_urlsafe(32)+'.bin'},
+    {'dir': 'excluded/', 'file': token_urlsafe(32)+'.txt'}
+])
+def test_select_excluded_files(context, path):
+    context.manager.workdir = context.join(path['dir'])
+    context.manager.exclude = ['*/excluded/*', '*.bin']
+    Path(context.join(path['dir']+path['file'])).touch()
+    assert path['file'] not in context.manager.files
+    with pytest.raises(OSError):
+        context.manager.filename = path['file']
+
+@pytest.mark.parametrize('path', [
+    {'src': 'excluded/empty.txt', 'dst': 'allowed/link.txt'},
+    {'src': 'invis/empty.txt', 'dst': 'allowed/link.txt'},
+    {'src': 'excluded/empty.txt', 'dst': 'allowed/'+token_urlsafe(16)+'.txt'},
+    {'src': 'invis/empty.txt', 'dst': 'allowed/'+token_urlsafe(16)+'.txt'},
+    {'src': 'excluded/empty.txt', 'dst': 'allowed/'+token_urlsafe(32)+'.txt'},
+    {'src': 'invis/empty.txt', 'dst': 'allowed/'+token_urlsafe(32)+'.txt'}
+])
+def test_select_excluded_symlinks(context, path):
+    symlink(context.join(path['src']), context.join(path['dst']))
+    assert path['dst'] not in context.manager.files
+
+def test_read_write_file(context):
+    for _ in range(128):
+        context.manager.filename = 'empty.txt'
+        content = token_urlsafe(32)
+        context.manager.file_contents = content
+        assert context.manager.file_contents == content
+        with open(context.join('allowed/empty.txt'), 'r') as ifile:
+            assert ifile.read() == content
+
+def test_regular_ide_actions(context):
+    context.manager.workdir = context.join('allowed')
+    newfile1, newfile2 = token_urlsafe(16), token_urlsafe(16)
+    Path(context.join(f'allowed/{newfile1}')).touch()
+    Path(context.join(f'allowed/{newfile2}')).touch()
+    for _ in range(8):
+        context.manager.filename = newfile1
+        content1 = token_urlsafe(32)
+        context.manager.file_contents = content1
+        context.manager.filename = newfile2
+        content2 = token_urlsafe(32)
+        context.manager.file_contents = content2
+        context.manager.filename = newfile1
+        assert context.manager.file_contents == content1
+        context.manager.filename = newfile2
+        assert context.manager.file_contents == content2

tfw/components/ide/ide_handler.py

@@ -0,0 +1,196 @@
+import logging
+
+from tfw.internals.networking import Scope
+from tfw.internals.inotify import InotifyObserver
+
+from .file_manager import FileManager
+
+
+LOG = logging.getLogger(__name__)
+
+BUILD_ARTIFACTS = (
+    "*.a",
+    "*.class",
+    "*.dll",
+    "*.dylib",
+    "*.elf",
+    "*.exe",
+    "*.jar",
+    "*.ko",
+    "*.la",
+    "*.lib",
+    "*.lo",
+    "*.o",
+    "*.obj",
+    "*.out",
+    "*.py[cod]",
+    "*.so",
+    "*.so.*",
+    "*.tar.gz",
+    "*.zip",
+    "*__pycache__*"
+)
+
+
+class IdeHandler:
+    keys = ['ide']
+    # pylint: disable=too-many-arguments,anomalous-backslash-in-string
+    """
+    Event handler implementing the backend of our browser based IDE.
+    By default all files in the directory specified in __init__ are displayed
+    on the fontend. Note that this is a stateful component.
+
+    When any file in the selected directory changes they are automatically refreshed
+    on the frontend (this is done by listening to inotify events).
+
+    This EventHandler accepts messages that have a data['command'] key specifying
+    a command to be executed.
+
+    The API of each command is documented in their respective handler.
+    """
+    def __init__(self, *, directory, allowed_directories, selected_file=None, exclude=None):
+        """
+        :param key: the key this instance should listen to
+        :param directory: working directory which the EventHandler should serve files from
+        :param allowed_directories: list of directories that can be switched to using selectdir
+        :param selected_file: file that is selected by default
+        :param exclude: list of filenames that should not appear between files (for .o, .pyc, etc.)
+        """
+        self.server_connector = None
+        try:
+            self.filemanager = FileManager(
+                allowed_directories=allowed_directories,
+                working_directory=directory,
+                selected_file=selected_file,
+                exclude=exclude
+            )
+        except IndexError:
+            raise EnvironmentError(
+                f'No file(s) in IdeEventHandler working_directory "{directory}"!'
+            )
+
+        self.monitor = InotifyObserver(
+            self.filemanager.allowed_directories,
+            exclude=BUILD_ARTIFACTS
+        )
+        self.monitor.on_modified = self._reload_frontend
+        self.monitor.start()
+
+        self.commands = {
+            'read':      self.read,
+            'write':     self.write,
+            'select':    self.select,
+            'selectdir': self.select_dir,
+            'exclude':   self.exclude
+        }
+
+    def _reload_frontend(self, event):  # pylint: disable=unused-argument
+        self.send_message({
+            'key': 'ide',
+            'data': {'command': 'reload'}
+        })
+
+    def send_message(self, message):
+        self.server_connector.send_message(message, scope=Scope.WEBSOCKET)
+
+    def read(self, data):
+        """
+        Read the currently selected file.
+
+        :return dict: TFW message data containing key 'content'
+                      (contents of the selected file)
+        """
+        try:
+            data['content'] = self.filemanager.file_contents
+        except PermissionError:
+            data['content'] = 'You have no permission to open that file :('
+        except FileNotFoundError:
+            data['content'] = 'This file was removed :('
+        except Exception: # pylint: disable=broad-except
+            data['content'] = 'Failed to read file :('
+        return data
+
+    def write(self, data):
+        """
+        Overwrites a file with the desired string.
+
+        :param data: TFW message data containing key 'content'
+                     (new file content)
+
+        """
+        try:
+            self.filemanager.file_contents = data['content']
+        except Exception: # pylint: disable=broad-except
+            LOG.exception('Error writing file!')
+        del data['content']
+        return data
+
+    def select(self, data):
+        """
+        Selects a file from the current directory.
+
+        :param data: TFW message data containing 'filename'
+                     (name of file to select relative to the current directory)
+        """
+        try:
+            self.filemanager.filename = data['filename']
+        except EnvironmentError:
+            LOG.exception('Failed to select file "%s"', data['filename'])
+        return data
+
+    def select_dir(self, data):
+        """
+        Select a new working directory to display files from.
+
+        :param data: TFW message data containing 'directory'
+                     (absolute path of diretory to select.
+                     must be a path whitelisted in
+                     self.allowed_directories)
+        """
+        try:
+            self.filemanager.workdir = data['directory']
+            try:
+                self.filemanager.filename = self.filemanager.files[0]
+                self.read(data)
+            except IndexError:
+                data['content'] = 'No files in this directory :('
+        except EnvironmentError as err:
+            LOG.error(
+                'Failed to select directory "%s". Reason: %s',
+                data['directory'], str(err)
+            )
+        return data
+
+    def exclude(self, data):
+        """
+        Overwrite list of excluded files
+
+        :param data: TFW message data containing 'exclude'
+                     (list of unix-style filename patterns to be excluded,
+                     e.g.: ["\*.pyc", "\*.o")
+        """
+        try:
+            self.filemanager.exclude = list(data['exclude'])
+        except TypeError:
+            LOG.error('Exclude must be Iterable!')
+        return data
+
+    def attach_fileinfo(self, data):
+        """
+        Basic information included in every response to the frontend.
+        """
+        data['filename'] = self.filemanager.filename
+        data['files'] = self.filemanager.files
+        data['directory'] = self.filemanager.workdir
+
+    def handle_event(self, message, _):
+        try:
+            data = message['data']
+            message['data'] = self.commands[data['command']](data)
+            self.attach_fileinfo(data)
+            self.send_message(message)
+        except KeyError:
+            LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)
+
+    def cleanup(self):
+        self.monitor.stop()