From ea3cd2c1c7b9ee016f1c5ba489addf9f52216892 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 29 Jun 2014 00:52:23 +0400 Subject: [PATCH] Add libuv-based watcher Fixes #821 --- powerline/lib/watcher/__init__.py | 10 ++ powerline/lib/watcher/tree.py | 7 ++ powerline/lib/watcher/uv.py | 167 ++++++++++++++++++++++++++++++ 3 files changed, 184 insertions(+) create mode 100644 powerline/lib/watcher/uv.py diff --git a/powerline/lib/watcher/__init__.py b/powerline/lib/watcher/__init__.py index a36df182..62894695 100644 --- a/powerline/lib/watcher/__init__.py +++ b/powerline/lib/watcher/__init__.py @@ -6,6 +6,7 @@ import sys from powerline.lib.watcher.stat import StatFileWatcher from powerline.lib.watcher.inotify import INotifyFileWatcher from powerline.lib.watcher.tree import TreeWatcher +from powerline.lib.watcher.uv import UvFileWatcher, UvNotFound from powerline.lib.inotify import INotifyError @@ -39,6 +40,9 @@ def create_file_watcher(pl, watcher_type='auto', expire_time=10): # Explicitly selected inotify watcher: do not catch INotifyError then. pl.debug('Using requested inotify watcher', prefix='watcher') return INotifyFileWatcher(expire_time=expire_time) + elif watcher_type == 'uv': + pl.debug('Using requested uv watcher', prefix='watcher') + return UvFileWatcher() if sys.platform.startswith('linux'): try: @@ -47,6 +51,12 @@ def create_file_watcher(pl, watcher_type='auto', expire_time=10): except INotifyError: pl.info('Failed to create inotify watcher', prefix='watcher') + try: + pl.debug('Using libuv-based watcher') + return UvFileWatcher() + except UvNotFound: + pl.debug('Failed to import pyuv') + pl.debug('Using stat-based watcher') return StatFileWatcher() diff --git a/powerline/lib/watcher/tree.py b/powerline/lib/watcher/tree.py index 127261f7..e8efba4e 100644 --- a/powerline/lib/watcher/tree.py +++ b/powerline/lib/watcher/tree.py @@ -7,6 +7,7 @@ from powerline.lib.monotonic import monotonic from powerline.lib.inotify import INotifyError from powerline.lib.path import realpath from powerline.lib.watcher.inotify import INotifyTreeWatcher, DirTooLarge, NoSuchDir, BaseDirChanged +from powerline.lib.watcher.uv import UvTreeWatcher, UvNotFound class DummyTreeWatcher(object): @@ -30,6 +31,8 @@ class TreeWatcher(object): def get_watcher(self, path, ignore_event): if self.watcher_type == 'inotify': return INotifyTreeWatcher(path, ignore_event=ignore_event) + if self.watcher_type == 'uv': + return UvTreeWatcher(path, ignore_event=ignore_event) if self.watcher_type == 'dummy': return DummyTreeWatcher(path) # FIXME @@ -42,6 +45,10 @@ class TreeWatcher(object): except (INotifyError, DirTooLarge) as e: if not isinstance(e, INotifyError): self.pl.warn('Failed to watch path: {0} with error: {1}'.format(path, e)) + try: + return UvTreeWatcher(path, ignore_event=ignore_event) + except UvNotFound: + pass return DummyTreeWatcher(path) else: raise ValueError('Unknown watcher type: {0}'.format(self.watcher_type)) diff --git a/powerline/lib/watcher/uv.py b/powerline/lib/watcher/uv.py new file mode 100644 index 00000000..16e6aeb7 --- /dev/null +++ b/powerline/lib/watcher/uv.py @@ -0,0 +1,167 @@ +# vim:fileencoding=utf-8:noet +from __future__ import (unicode_literals, absolute_import, print_function) + +from powerline.lib.path import realpath + +from collections import defaultdict +from threading import RLock +from functools import partial +from threading import Thread + +import os + + +class UvNotFound(NotImplementedError): + pass + + +pyuv = None + + +def import_pyuv(): + global pyuv + if not pyuv: + try: + pyuv = __import__('pyuv') + except ImportError: + raise UvNotFound + + +class UvThread(Thread): + daemon = True + + def __init__(self, loop): + self.uv_loop = loop + super(UvThread, self).__init__() + + def run(self): + while True: + self.uv_loop.run() + + def join(self): + self.uv_loop.stop() + return super(UvThread, self).join() + + +_uv_thread = None + + +def start_uv_thread(): + global _uv_thread + if _uv_thread is None: + loop = pyuv.Loop() + _uv_thread = UvThread(loop) + _uv_thread.start() + return _uv_thread.uv_loop + + +class UvWatcher(object): + def __init__(self): + import_pyuv() + self.watches = {} + self.lock = RLock() + self.loop = start_uv_thread() + + def watch(self, path): + with self.lock: + if path not in self.watches: + try: + self.watches[path] = pyuv.fs.FSEvent( + self.loop, + path, + partial(self._record_event, path), + pyuv.fs.UV_CHANGE | pyuv.fs.UV_RENAME + ) + except pyuv.error.FSEventError as e: + code = e.args[0] + if code == pyuv.errno.UV_ENOENT: + raise OSError('No such file or directory: ' + path) + else: + raise + + def unwatch(self, path): + with self.lock: + try: + watch = self.watches.pop(path) + except KeyError: + return + watch.close(partial(self._stopped_watching, path)) + + def __del__(self): + try: + lock = self.lock + except AttributeError: + pass + else: + with lock: + while self.watches: + path, watch = self.watches.popitem() + watch.close(partial(self._stopped_watching, path)) + + +class UvFileWatcher(UvWatcher): + def __init__(self): + super(UvFileWatcher, self).__init__() + self.events = defaultdict(list) + + def _record_event(self, path, fsevent_handle, filename, events, error): + with self.lock: + self.events[path].append(events) + if events | pyuv.fs.UV_RENAME: + if not os.path.exists(path): + self.watches.pop(path).close() + + def _stopped_watching(self, path, *args): + self.events.pop(path, None) + + def __call__(self, path): + with self.lock: + events = self.events.pop(path, None) + if events: + return True + if path not in self.watches: + self.watch(path) + return True + + +class UvTreeWatcher(UvWatcher): + is_dummy = False + + def __init__(self, basedir, ignore_event=None): + super(UvTreeWatcher, self).__init__() + self.ignore_event = ignore_event or (lambda path, name: False) + self.basedir = realpath(basedir) + self.modified = True + self.watch_directory(self.basedir) + + def watch_directory(self, path): + os.path.walk(path, self.watch_one_directory, None) + + def watch_one_directory(self, arg, dirname, fnames): + try: + self.watch(dirname) + except OSError: + pass + + def _stopped_watching(self, path, *args): + pass + + def _record_event(self, path, fsevent_handle, filename, events, error): + if not self.ignore_event(path, filename): + self.modified = True + if events == pyuv.fs.UV_CHANGE | pyuv.fs.UV_RENAME: + # Stat changes to watched directory are UV_CHANGE|UV_RENAME. It + # is weird. + pass + elif events | pyuv.fs.UV_RENAME: + if not os.path.isdir(path): + self.unwatch(path) + else: + full_name = os.path.join(path, filename) + if os.path.isdir(full_name): + # For some reason mkdir and rmdir both fall into this + # category + self.watch_directory(full_name) + + def __call__(self): + return self.__dict__.pop('modified', False)