Simplify IDE handler and file manager

tfw/components/ide/file_manager/file_manager.py

@@ -1,93 +1,56 @@
-from typing import Iterable
+from functools import wraps
 from glob import glob
 from fnmatch import fnmatchcase
-from os.path import basename, isfile, join, relpath, exists, isdir, realpath
+from os.path import dirname, isdir, isfile, realpath
+
+
+def _with_is_allowed(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        if self._is_allowed(args[0]): # pylint: disable=protected-access
+            return func(self, *args, **kwargs)
+        raise ValueError('Forbidden path.')
+    return wrapper
 
 
 class FileManager: # pylint: disable=too-many-instance-attributes
-    def __init__(self, working_directory, allowed_directories, selected_file=None, exclude=None):
+    def __init__(self, patterns):
-        self._exclude, self.exclude = [], exclude
+        self.patterns = patterns
-        self._allowed_directories, self.allowed_directories = None, allowed_directories
-        self._workdir, self.workdir = None, working_directory
-        self._filename, self.filename = None, selected_file or self.files[0]
-
-    @property
-    def exclude(self):
-        return self._exclude
-
-    @exclude.setter
-    def exclude(self, exclude):
-        if exclude is None:
-            return
-        if not isinstance(exclude, Iterable):
-            raise TypeError('Exclude must be Iterable!')
-        self._exclude = exclude
-
-    @property
-    def workdir(self):
-        return self._workdir
-
-    @workdir.setter
-    def workdir(self, directory):
-        if not exists(directory) or not isdir(directory):
-            raise EnvironmentError(f'"{directory}" is not a directory!')
-        if not self._is_in_allowed_dir(directory):
-            raise EnvironmentError(f'Directory "{directory}" is not allowed!')
-        self._workdir = directory
-
-    @property
-    def allowed_directories(self):
-        return self._allowed_directories
-
-    @allowed_directories.setter
-    def allowed_directories(self, directories):
-        self._allowed_directories = [realpath(directory) for directory in directories]
-
-    @property
-    def filename(self):
-        return self._filename
-
-    @filename.setter
-    def filename(self, filename):
-        if filename not in self.files:
-            raise EnvironmentError('No such file in workdir!')
-        self._filename = filename
 
     @property
     def files(self):
-        return [
+        return list(set(
-            self._relpath(file)
+            path
-            for file in glob(join(self._workdir, '**/*'), recursive=True)
+            for pattern in self.patterns
-            if isfile(file)
+            for path in glob(pattern, recursive=True)
-            and self._is_in_allowed_dir(file)
+            if isfile(path) and self._is_allowed(path)
-            and not self._is_blacklisted(file)
+        ))
-        ]
 
     @property
-    def file_contents(self):
+    def parents(self):
-        with open(self._filepath(self.filename), 'rb', buffering=0) as ifile:
+        return list(set(
+            self._find_directory(pattern)
+            for pattern in self.patterns
+        ))
+
+    @staticmethod
+    def _find_directory(pattern):
+        while pattern and not isdir(pattern):
+            pattern = dirname(pattern)
+        return pattern
+
+    def _is_allowed(self, filepath):
+        return any(
+            fnmatchcase(realpath(filepath), pattern)
+            for pattern in self.patterns
+        )
+
+    @_with_is_allowed
+    def read_file(self, filepath): # pylint: disable=no-self-use
+        with open(filepath, 'rb', buffering=0) as ifile:
             return ifile.read().decode(errors='surrogateescape')
 
-    @file_contents.setter
+    @_with_is_allowed
-    def file_contents(self, value):
+    def write_file(self, filepath, contents): # pylint: disable=no-self-use
-        with open(self._filepath(self.filename), 'wb', buffering=0) as ofile:
+        with open(filepath, 'wb', buffering=0) as ofile:
-            ofile.write(value.encode())
+            ofile.write(contents.encode())
-
-    def _is_in_allowed_dir(self, path):
-        return any(
-            realpath(path).startswith(allowed_dir)
-            for allowed_dir in self.allowed_directories
-        )
-
-    def _is_blacklisted(self, file):
-        return any(
-            fnmatchcase(file, blacklisted) or
-            fnmatchcase(basename(file), blacklisted)
-            for blacklisted in self.exclude
-        )
-
-    def _filepath(self, filename):
-        return join(self._workdir, filename)
-
-    def _relpath(self, filename):
-        return relpath(self._filepath(filename), start=self._workdir)

tfw/components/ide/file_manager/test_file_manager.py

@@ -1,8 +1,8 @@
 # pylint: disable=redefined-outer-name
 from dataclasses import dataclass
 from secrets import token_urlsafe
+from os import mkdir, symlink
 from os.path import join
-from os import chdir, mkdir, symlink
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -13,112 +13,75 @@ from .file_manager import FileManager
 
 @dataclass
 class ManagerContext:
-    folder: str
+    workdir: str
+    subdir: str
+    subfile: str
     manager: FileManager
 
-    def join(self, path):
+    def create_random_file(self, dirname, extension):
-        return join(self.folder, path)
+        filename = self.join(f'{dirname}/{generate_name()}{extension}')
+        Path(filename).touch()
+        return filename
 
+    def create_random_folder(self, basepath):
+        dirname = self.join(f'{basepath}/{generate_name()}')
+        mkdir(dirname)
+        return dirname
+
+    def create_random_link(self, source, dirname, extension):
+        linkname = self.join(f'{dirname}/{generate_name()}{extension}')
+        symlink(source, linkname)
+        return linkname
+
+    def join(self, path):
+        return join(self.workdir, path)
+
+def generate_name():
+    return token_urlsafe(16)
 
 @pytest.fixture()
 def context():
-    dirs = {}
-
     with TemporaryDirectory() as workdir:
-        chdir(workdir)
+        subdir = join(workdir, generate_name())
-        for name in ['allowed', 'excluded', 'invis']:
+        subfile = join(subdir, generate_name() + '.txt')
-            node = join(workdir, name)
+        mkdir(subdir)
-            mkdir(node)
+        Path(subfile).touch()
-            Path(join(node, 'empty.txt')).touch()
+        manager = FileManager([join(workdir, '**/*.txt')])
-            Path(join(node, 'empty.bin')).touch()
+        yield ManagerContext(workdir, subdir, subfile, manager)
-            dirs[name] = node
 
-        yield ManagerContext(
+def test_matching_files(context):
-            workdir,
+    newdir = context.create_random_folder(context.subdir)
-            FileManager(
+    newfile = context.create_random_file(newdir, '.txt')
-                dirs['allowed'],
+    newlink = context.create_random_link(newfile, newdir, '.txt')
-                [dirs['allowed'], dirs['excluded']],
+    assert set(context.manager.files) == {context.subfile, newfile, newlink}
-                exclude=['*/excluded/*']
-            )
-        )
 
-@pytest.mark.parametrize('subdir', ['allowed/', 'excluded/'])
+def test_unmatching_files(context):
-def test_select_allowed_dirs(context, subdir):
+    newtxt = context.create_random_file(context.workdir, '.txt')
-    context.manager.workdir = context.join(subdir)
+    newbin = context.create_random_file(context.subdir, '.bin')
-    assert context.manager.workdir == context.join(subdir)
+    context.create_random_link(newtxt, context.subdir, '.txt')
-    newdir = context.join(subdir+'deep')
+    context.create_random_link(newbin, context.subdir, '.txt')
-    mkdir(newdir)
+    assert context.manager.files == [context.subfile]
-    context.manager.workdir = newdir
-    assert context.manager.workdir == newdir
 
-@pytest.mark.parametrize('invdir', ['', 'invis'])
+def test_parents(context):
-def test_select_forbidden_dirs(context, invdir):
+    newdir = context.create_random_folder(context.workdir)
-    fullpath = context.join(invdir)
+    context.manager.patterns += [f'{newdir}/[!/@]*/**/?.c']
-    with pytest.raises(OSError):
+    assert set(context.manager.parents) == {context.workdir, newdir}
-        context.manager.workdir = fullpath
-    assert context.manager.workdir != fullpath
-    context.manager.allowed_directories += [fullpath]
-    context.manager.workdir = fullpath
-    assert context.manager.workdir == fullpath
-
-
-@pytest.mark.parametrize('filename', ['another.txt', '*.txt'])
-def test_select_allowed_files(context, filename):
-    Path(context.join('allowed/'+filename)).touch()
-    assert filename in context.manager.files
-    context.manager.filename = filename
-    assert context.manager.filename == filename
-
-@pytest.mark.parametrize('path', [
-    {'dir': 'allowed/', 'file': 'illegal.bin'},
-    {'dir': 'excluded/', 'file': 'legal.txt'},
-    {'dir': 'allowed/', 'file': token_urlsafe(16)+'.bin'},
-    {'dir': 'excluded/', 'file': token_urlsafe(16)+'.txt'},
-    {'dir': 'allowed/', 'file': token_urlsafe(32)+'.bin'},
-    {'dir': 'excluded/', 'file': token_urlsafe(32)+'.txt'}
-])
-def test_select_excluded_files(context, path):
-    context.manager.workdir = context.join(path['dir'])
-    context.manager.exclude = ['*/excluded/*', '*.bin']
-    Path(context.join(path['dir']+path['file'])).touch()
-    assert path['file'] not in context.manager.files
-    with pytest.raises(OSError):
-        context.manager.filename = path['file']
-
-@pytest.mark.parametrize('path', [
-    {'src': 'excluded/empty.txt', 'dst': 'allowed/link.txt'},
-    {'src': 'invis/empty.txt', 'dst': 'allowed/link.txt'},
-    {'src': 'excluded/empty.txt', 'dst': 'allowed/'+token_urlsafe(16)+'.txt'},
-    {'src': 'invis/empty.txt', 'dst': 'allowed/'+token_urlsafe(16)+'.txt'},
-    {'src': 'excluded/empty.txt', 'dst': 'allowed/'+token_urlsafe(32)+'.txt'},
-    {'src': 'invis/empty.txt', 'dst': 'allowed/'+token_urlsafe(32)+'.txt'}
-])
-def test_select_excluded_symlinks(context, path):
-    symlink(context.join(path['src']), context.join(path['dst']))
-    assert path['dst'] not in context.manager.files
 
 def test_read_write_file(context):
     for _ in range(128):
-        context.manager.filename = 'empty.txt'
         content = token_urlsafe(32)
-        context.manager.file_contents = content
+        context.manager.write_file(context.subfile, content)
-        assert context.manager.file_contents == content
+        assert context.manager.read_file(context.subfile) == content
-        with open(context.join('allowed/empty.txt'), 'r') as ifile:
+        with open(context.subfile, 'r') as ifile:
             assert ifile.read() == content
 
 def test_regular_ide_actions(context):
-    context.manager.workdir = context.join('allowed')
+    newfile1 = context.create_random_file(context.subdir, '.txt')
-    newfile1, newfile2 = token_urlsafe(16), token_urlsafe(16)
+    newfile2 = context.create_random_file(context.subdir, '.txt')
-    Path(context.join(f'allowed/{newfile1}')).touch()
+    for _ in range(4):
-    Path(context.join(f'allowed/{newfile2}')).touch()
-    for _ in range(8):
         context.manager.filename = newfile1
-        content1 = token_urlsafe(32)
+        content1, content2 = token_urlsafe(32), token_urlsafe(32)
-        context.manager.file_contents = content1
+        context.manager.write_file(newfile1, content1)
-        context.manager.filename = newfile2
+        context.manager.write_file(newfile2, content2)
-        content2 = token_urlsafe(32)
+        assert context.manager.read_file(newfile1) == content1
-        context.manager.file_contents = content2
+        assert context.manager.read_file(newfile2) == content2
-        context.manager.filename = newfile1
-        assert context.manager.file_contents == content1
-        context.manager.filename = newfile2
-        assert context.manager.file_contents == content2

tfw/components/ide/ide_handler.py

@@ -1,4 +1,5 @@
 import logging
+from os.path import isfile
 
 from tfw.internals.networking import Scope
 from tfw.internals.inotify import InotifyObserver
@@ -32,161 +33,62 @@ BUILD_ARTIFACTS = (
 
 
 class IdeHandler:
-    keys = ['ide']
+    keys = ['ide.read', 'ide.write']
-    # pylint: disable=too-many-arguments,anomalous-backslash-in-string
-    """
-    Event handler implementing the backend of our browser based IDE.
-    By default all files in the directory specified in __init__ are displayed
-    on the fontend. Note that this is a stateful component.
 
-    When any file in the selected directory changes they are automatically refreshed
+    def __init__(self, *, patterns, initial_file=''):
-    on the frontend (this is done by listening to inotify events).
-
-    This EventHandler accepts messages that have a data['command'] key specifying
-    a command to be executed.
-
-    The API of each command is documented in their respective handler.
-    """
-    def __init__(self, *, directory, allowed_directories, selected_file=None, exclude=None):
-        """
-        :param key: the key this instance should listen to
-        :param directory: working directory which the EventHandler should serve files from
-        :param allowed_directories: list of directories that can be switched to using selectdir
-        :param selected_file: file that is selected by default
-        :param exclude: list of filenames that should not appear between files (for .o, .pyc, etc.)
-        """
         self.connector = None
-        try:
+        self.filemanager = FileManager(patterns)
-            self.filemanager = FileManager(
+        self._initial_file = initial_file
-                allowed_directories=allowed_directories,
-                working_directory=directory,
-                selected_file=selected_file,
-                exclude=exclude
-            )
-        except IndexError:
-            raise EnvironmentError(
-                f'No file(s) in IdeEventHandler working_directory "{directory}"!'
-            )
 
         self.monitor = InotifyObserver(
-            self.filemanager.allowed_directories,
+            path=self.filemanager.parents,
             exclude=BUILD_ARTIFACTS
         )
         self.monitor.on_modified = self._reload_frontend
         self.monitor.start()
 
         self.commands = {
-            'read':      self.read,
+            'ide.read'  : self.read,
-            'write':     self.write,
+            'ide.write' : self.write
-            'select':    self.select,
-            'selectdir': self.select_dir,
-            'exclude':   self.exclude
         }
 
+    @property
+    def initial_file(self):
+        if not isfile(self._initial_file):
+            self._initial_file = self.filemanager.files[0]
+        return self._initial_file
+
     def _reload_frontend(self, event):  # pylint: disable=unused-argument
-        self.send_message({
+        self.send_message({'key': 'ide.reload'})
-            'key': 'ide',
-            'data': {'command': 'reload'}
-        })
 
     def send_message(self, message):
         self.connector.send_message(message, scope=Scope.WEBSOCKET)
 
-    def read(self, data):
+    def read(self, message):
-        """
+        if message.get('files'):
-        Read the currently selected file.
+            self.filemanager.patterns = message['files']
-
-        :return dict: TFW message data containing key 'content'
-                      (contents of the selected file)
-        """
         try:
-            data['content'] = self.filemanager.file_contents
+            message['content'] = self.filemanager.read_file(message['filename'])
         except PermissionError:
-            data['content'] = 'You have no permission to open that file :('
+            message['content'] = 'You have no permission to open that file :('
         except FileNotFoundError:
-            data['content'] = 'This file was removed :('
+            message['content'] = 'This file was removed :('
         except Exception: # pylint: disable=broad-except
-            data['content'] = 'Failed to read file :('
+            message['content'] = 'Failed to read file :('
-        return data
 
-    def write(self, data):
+    def write(self, message):
-        """
-        Overwrites a file with the desired string.
-
-        :param data: TFW message data containing key 'content'
-                     (new file content)
-
-        """
         try:
-            self.filemanager.file_contents = data['content']
+            self.filemanager.write_file(message['filename'], message['content'])
         except Exception: # pylint: disable=broad-except
             LOG.exception('Error writing file!')
-        del data['content']
+        del message['content']
-        return data
-
-    def select(self, data):
-        """
-        Selects a file from the current directory.
-
-        :param data: TFW message data containing 'filename'
-                     (name of file to select relative to the current directory)
-        """
-        try:
-            self.filemanager.filename = data['filename']
-        except EnvironmentError:
-            LOG.exception('Failed to select file "%s"', data['filename'])
-        return data
-
-    def select_dir(self, data):
-        """
-        Select a new working directory to display files from.
-
-        :param data: TFW message data containing 'directory'
-                     (absolute path of diretory to select.
-                     must be a path whitelisted in
-                     self.allowed_directories)
-        """
-        try:
-            self.filemanager.workdir = data['directory']
-            try:
-                self.filemanager.filename = self.filemanager.files[0]
-                self.read(data)
-            except IndexError:
-                data['content'] = 'No files in this directory :('
-        except EnvironmentError as err:
-            LOG.error(
-                'Failed to select directory "%s". Reason: %s',
-                data['directory'], str(err)
-            )
-        return data
-
-    def exclude(self, data):
-        """
-        Overwrite list of excluded files
-
-        :param data: TFW message data containing 'exclude'
-                     (list of unix-style filename patterns to be excluded,
-                     e.g.: ["\*.pyc", "\*.o")
-        """
-        try:
-            self.filemanager.exclude = list(data['exclude'])
-        except TypeError:
-            LOG.error('Exclude must be Iterable!')
-        return data
-
-    def attach_fileinfo(self, data):
-        """
-        Basic information included in every response to the frontend.
-        """
-        data['filename'] = self.filemanager.filename
-        data['files'] = self.filemanager.files
-        data['directory'] = self.filemanager.workdir
 
     def handle_event(self, message, _):
         try:
-            data = message['data']
+            if message['filename'] == '':
-            message['data'] = self.commands[data['command']](data)
+                message['filename'] = self.initial_file
-            self.attach_fileinfo(data)
+            self.commands[message['key']](message)
+            message['files'] = self.filemanager.files
             self.send_message(message)
         except KeyError:
             LOG.error('IGNORING MESSAGE: Invalid message received: %s', message)