Nikolai Aleksandrovich Pavlov
commit
d714fb0d88
|
|
@ -6,6 +6,7 @@ import sys
|
|||
from powerline.lib.watcher.stat import StatFileWatcher
|
||||
from powerline.lib.watcher.inotify import INotifyFileWatcher
|
||||
from powerline.lib.watcher.tree import TreeWatcher
|
||||
from powerline.lib.watcher.uv import UvFileWatcher, UvNotFound
|
||||
from powerline.lib.inotify import INotifyError
|
||||
|
||||
|
||||
|
|
@ -39,6 +40,9 @@ def create_file_watcher(pl, watcher_type='auto', expire_time=10):
|
|||
# Explicitly selected inotify watcher: do not catch INotifyError then.
|
||||
pl.debug('Using requested inotify watcher', prefix='watcher')
|
||||
return INotifyFileWatcher(expire_time=expire_time)
|
||||
elif watcher_type == 'uv':
|
||||
pl.debug('Using requested uv watcher', prefix='watcher')
|
||||
return UvFileWatcher()
|
||||
|
||||
if sys.platform.startswith('linux'):
|
||||
try:
|
||||
|
|
@ -47,6 +51,12 @@ def create_file_watcher(pl, watcher_type='auto', expire_time=10):
|
|||
except INotifyError:
|
||||
pl.info('Failed to create inotify watcher', prefix='watcher')
|
||||
|
||||
try:
|
||||
pl.debug('Using libuv-based watcher')
|
||||
return UvFileWatcher()
|
||||
except UvNotFound:
|
||||
pl.debug('Failed to import pyuv')
|
||||
|
||||
pl.debug('Using stat-based watcher')
|
||||
return StatFileWatcher()
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from powerline.lib.monotonic import monotonic
|
|||
from powerline.lib.inotify import INotifyError
|
||||
from powerline.lib.path import realpath
|
||||
from powerline.lib.watcher.inotify import INotifyTreeWatcher, DirTooLarge, NoSuchDir, BaseDirChanged
|
||||
from powerline.lib.watcher.uv import UvTreeWatcher, UvNotFound
|
||||
|
||||
|
||||
class DummyTreeWatcher(object):
|
||||
|
|
@ -30,6 +31,8 @@ class TreeWatcher(object):
|
|||
def get_watcher(self, path, ignore_event):
|
||||
if self.watcher_type == 'inotify':
|
||||
return INotifyTreeWatcher(path, ignore_event=ignore_event)
|
||||
if self.watcher_type == 'uv':
|
||||
return UvTreeWatcher(path, ignore_event=ignore_event)
|
||||
if self.watcher_type == 'dummy':
|
||||
return DummyTreeWatcher(path)
|
||||
# FIXME
|
||||
|
|
@ -42,6 +45,10 @@ class TreeWatcher(object):
|
|||
except (INotifyError, DirTooLarge) as e:
|
||||
if not isinstance(e, INotifyError):
|
||||
self.pl.warn('Failed to watch path: {0} with error: {1}'.format(path, e))
|
||||
try:
|
||||
return UvTreeWatcher(path, ignore_event=ignore_event)
|
||||
except UvNotFound:
|
||||
pass
|
||||
return DummyTreeWatcher(path)
|
||||
else:
|
||||
raise ValueError('Unknown watcher type: {0}'.format(self.watcher_type))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,167 @@
|
|||
# vim:fileencoding=utf-8:noet
|
||||
from __future__ import (unicode_literals, absolute_import, print_function)
|
||||
|
||||
from powerline.lib.path import realpath
|
||||
|
||||
from collections import defaultdict
|
||||
from threading import RLock
|
||||
from functools import partial
|
||||
from threading import Thread
|
||||
|
||||
import os
|
||||
|
||||
|
||||
class UvNotFound(NotImplementedError):
|
||||
pass
|
||||
|
||||
|
||||
pyuv = None
|
||||
|
||||
|
||||
def import_pyuv():
|
||||
global pyuv
|
||||
if not pyuv:
|
||||
try:
|
||||
pyuv = __import__('pyuv')
|
||||
except ImportError:
|
||||
raise UvNotFound
|
||||
|
||||
|
||||
class UvThread(Thread):
|
||||
daemon = True
|
||||
|
||||
def __init__(self, loop):
|
||||
self.uv_loop = loop
|
||||
super(UvThread, self).__init__()
|
||||
|
||||
def run(self):
|
||||
while True:
|
||||
self.uv_loop.run()
|
||||
|
||||
def join(self):
|
||||
self.uv_loop.stop()
|
||||
return super(UvThread, self).join()
|
||||
|
||||
|
||||
_uv_thread = None
|
||||
|
||||
|
||||
def start_uv_thread():
|
||||
global _uv_thread
|
||||
if _uv_thread is None:
|
||||
loop = pyuv.Loop()
|
||||
_uv_thread = UvThread(loop)
|
||||
_uv_thread.start()
|
||||
return _uv_thread.uv_loop
|
||||
|
||||
|
||||
class UvWatcher(object):
|
||||
def __init__(self):
|
||||
import_pyuv()
|
||||
self.watches = {}
|
||||
self.lock = RLock()
|
||||
self.loop = start_uv_thread()
|
||||
|
||||
def watch(self, path):
|
||||
with self.lock:
|
||||
if path not in self.watches:
|
||||
try:
|
||||
self.watches[path] = pyuv.fs.FSEvent(
|
||||
self.loop,
|
||||
path,
|
||||
partial(self._record_event, path),
|
||||
pyuv.fs.UV_CHANGE | pyuv.fs.UV_RENAME
|
||||
)
|
||||
except pyuv.error.FSEventError as e:
|
||||
code = e.args[0]
|
||||
if code == pyuv.errno.UV_ENOENT:
|
||||
raise OSError('No such file or directory: ' + path)
|
||||
else:
|
||||
raise
|
||||
|
||||
def unwatch(self, path):
|
||||
with self.lock:
|
||||
try:
|
||||
watch = self.watches.pop(path)
|
||||
except KeyError:
|
||||
return
|
||||
watch.close(partial(self._stopped_watching, path))
|
||||
|
||||
def __del__(self):
|
||||
try:
|
||||
lock = self.lock
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
with lock:
|
||||
while self.watches:
|
||||
path, watch = self.watches.popitem()
|
||||
watch.close(partial(self._stopped_watching, path))
|
||||
|
||||
|
||||
class UvFileWatcher(UvWatcher):
|
||||
def __init__(self):
|
||||
super(UvFileWatcher, self).__init__()
|
||||
self.events = defaultdict(list)
|
||||
|
||||
def _record_event(self, path, fsevent_handle, filename, events, error):
|
||||
with self.lock:
|
||||
self.events[path].append(events)
|
||||
if events | pyuv.fs.UV_RENAME:
|
||||
if not os.path.exists(path):
|
||||
self.watches.pop(path).close()
|
||||
|
||||
def _stopped_watching(self, path, *args):
|
||||
self.events.pop(path, None)
|
||||
|
||||
def __call__(self, path):
|
||||
with self.lock:
|
||||
events = self.events.pop(path, None)
|
||||
if events:
|
||||
return True
|
||||
if path not in self.watches:
|
||||
self.watch(path)
|
||||
return True
|
||||
|
||||
|
||||
class UvTreeWatcher(UvWatcher):
|
||||
is_dummy = False
|
||||
|
||||
def __init__(self, basedir, ignore_event=None):
|
||||
super(UvTreeWatcher, self).__init__()
|
||||
self.ignore_event = ignore_event or (lambda path, name: False)
|
||||
self.basedir = realpath(basedir)
|
||||
self.modified = True
|
||||
self.watch_directory(self.basedir)
|
||||
|
||||
def watch_directory(self, path):
|
||||
os.path.walk(path, self.watch_one_directory, None)
|
||||
|
||||
def watch_one_directory(self, arg, dirname, fnames):
|
||||
try:
|
||||
self.watch(dirname)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _stopped_watching(self, path, *args):
|
||||
pass
|
||||
|
||||
def _record_event(self, path, fsevent_handle, filename, events, error):
|
||||
if not self.ignore_event(path, filename):
|
||||
self.modified = True
|
||||
if events == pyuv.fs.UV_CHANGE | pyuv.fs.UV_RENAME:
|
||||
# Stat changes to watched directory are UV_CHANGE|UV_RENAME. It
|
||||
# is weird.
|
||||
pass
|
||||
elif events | pyuv.fs.UV_RENAME:
|
||||
if not os.path.isdir(path):
|
||||
self.unwatch(path)
|
||||
else:
|
||||
full_name = os.path.join(path, filename)
|
||||
if os.path.isdir(full_name):
|
||||
# For some reason mkdir and rmdir both fall into this
|
||||
# category
|
||||
self.watch_directory(full_name)
|
||||
|
||||
def __call__(self):
|
||||
return self.__dict__.pop('modified', False)
|
||||
|
|
@ -4,6 +4,7 @@ from __future__ import division
|
|||
import threading
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
|
||||
from time import sleep
|
||||
from subprocess import call, PIPE
|
||||
|
|
@ -581,12 +582,7 @@ class TestVCS(TestCase):
|
|||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
for repo_dir in [GIT_REPO] + ([HG_REPO] if use_mercurial else []) + ([BZR_REPO] if use_bzr else []):
|
||||
for root, dirs, files in list(os.walk(repo_dir, topdown=False)):
|
||||
for file in files:
|
||||
os.remove(os.path.join(root, file))
|
||||
for dir in dirs:
|
||||
os.rmdir(os.path.join(root, dir))
|
||||
os.rmdir(repo_dir)
|
||||
shutil.rmtree(repo_dir)
|
||||
if use_mercurial:
|
||||
if cls.powerline_old_HGRCPATH is None:
|
||||
os.environ.pop('HGRCPATH')
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
# vim:fileencoding=utf-8:noet
|
||||
from __future__ import absolute_import, unicode_literals, print_function, division
|
||||
|
||||
from powerline.lib.watcher import create_file_watcher, create_tree_watcher, INotifyError
|
||||
from powerline import get_fallback_logger
|
||||
from powerline.lib.monotonic import monotonic
|
||||
|
||||
import shutil
|
||||
import os
|
||||
|
||||
from time import sleep
|
||||
from functools import partial
|
||||
import os
|
||||
|
||||
from powerline.lib.watcher import create_file_watcher, create_tree_watcher, INotifyError
|
||||
from powerline.lib.watcher.uv import UvNotFound
|
||||
from powerline import get_fallback_logger
|
||||
from powerline.lib.monotonic import monotonic
|
||||
|
||||
from tests import TestCase, SkipTest
|
||||
|
||||
|
|
@ -16,6 +18,14 @@ from tests import TestCase, SkipTest
|
|||
INOTIFY_DIR = 'inotify' + os.environ.get('PYTHON', '')
|
||||
|
||||
|
||||
def clear_dir(dir):
|
||||
for root, dirs, files in list(os.walk(dir, topdown=False)):
|
||||
for f in files:
|
||||
os.remove(os.path.join(root, f))
|
||||
for d in dirs:
|
||||
os.rmdir(os.path.join(root, d))
|
||||
|
||||
|
||||
class TestFilesystemWatchers(TestCase):
|
||||
def do_test_for_change(self, watcher, path):
|
||||
st = monotonic()
|
||||
|
|
@ -30,78 +40,106 @@ class TestFilesystemWatchers(TestCase):
|
|||
w = create_file_watcher(pl=get_fallback_logger(), watcher_type='inotify')
|
||||
except INotifyError:
|
||||
raise SkipTest('This test is not suitable for a stat based file watcher')
|
||||
f1, f2, f3 = map(lambda x: os.path.join(INOTIFY_DIR, 'file%d' % x), (1, 2, 3))
|
||||
with open(f1, 'wb'):
|
||||
with open(f2, 'wb'):
|
||||
with open(f3, 'wb'):
|
||||
return self.do_test_file_watcher(w)
|
||||
|
||||
def do_test_file_watcher(self, w):
|
||||
try:
|
||||
f1, f2, f3 = map(lambda x: os.path.join(INOTIFY_DIR, 'file%d' % x), (1, 2, 3))
|
||||
with open(f1, 'wb'):
|
||||
with open(f2, 'wb'):
|
||||
with open(f3, 'wb'):
|
||||
pass
|
||||
ne = os.path.join(INOTIFY_DIR, 'notexists')
|
||||
self.assertRaises(OSError, w, ne)
|
||||
self.assertTrue(w(f1))
|
||||
self.assertTrue(w(f2))
|
||||
os.utime(f1, None), os.utime(f2, None)
|
||||
self.do_test_for_change(w, f1)
|
||||
self.do_test_for_change(w, f2)
|
||||
# Repeat once
|
||||
os.utime(f1, None), os.utime(f2, None)
|
||||
self.do_test_for_change(w, f1)
|
||||
self.do_test_for_change(w, f2)
|
||||
# Check that no false changes are reported
|
||||
self.assertFalse(w(f1), 'Spurious change detected')
|
||||
self.assertFalse(w(f2), 'Spurious change detected')
|
||||
# Check that open the file with 'w' triggers a change
|
||||
with open(f1, 'wb'):
|
||||
with open(f2, 'wb'):
|
||||
pass
|
||||
ne = os.path.join(INOTIFY_DIR, 'notexists')
|
||||
self.assertRaises(OSError, w, ne)
|
||||
self.assertTrue(w(f1))
|
||||
self.assertTrue(w(f2))
|
||||
os.utime(f1, None), os.utime(f2, None)
|
||||
self.do_test_for_change(w, f1)
|
||||
self.do_test_for_change(w, f2)
|
||||
# Repeat once
|
||||
os.utime(f1, None), os.utime(f2, None)
|
||||
self.do_test_for_change(w, f1)
|
||||
self.do_test_for_change(w, f2)
|
||||
# Check that no false changes are reported
|
||||
self.assertFalse(w(f1), 'Spurious change detected')
|
||||
self.assertFalse(w(f2), 'Spurious change detected')
|
||||
# Check that open the file with 'w' triggers a change
|
||||
with open(f1, 'wb'):
|
||||
with open(f2, 'wb'):
|
||||
pass
|
||||
self.do_test_for_change(w, f1)
|
||||
self.do_test_for_change(w, f2)
|
||||
# Check that writing to a file with 'a' triggers a change
|
||||
with open(f1, 'ab') as f:
|
||||
f.write(b'1')
|
||||
self.do_test_for_change(w, f1)
|
||||
# Check that deleting a file registers as a change
|
||||
os.unlink(f1)
|
||||
self.do_test_for_change(w, f1)
|
||||
# Test that changing the inode of a file does not cause it to stop
|
||||
# being watched
|
||||
os.rename(f3, f2)
|
||||
self.do_test_for_change(w, f2)
|
||||
self.assertFalse(w(f2), 'Spurious change detected')
|
||||
os.utime(f2, None)
|
||||
self.do_test_for_change(w, f2)
|
||||
self.do_test_for_change(w, f1)
|
||||
self.do_test_for_change(w, f2)
|
||||
# Check that writing to a file with 'a' triggers a change
|
||||
with open(f1, 'ab') as f:
|
||||
f.write(b'1')
|
||||
self.do_test_for_change(w, f1)
|
||||
# Check that deleting a file registers as a change
|
||||
os.unlink(f1)
|
||||
self.do_test_for_change(w, f1)
|
||||
# Test that changing the inode of a file does not cause it to stop
|
||||
# being watched
|
||||
os.rename(f3, f2)
|
||||
self.do_test_for_change(w, f2)
|
||||
self.assertFalse(w(f2), 'Spurious change detected')
|
||||
os.utime(f2, None)
|
||||
self.do_test_for_change(w, f2)
|
||||
finally:
|
||||
clear_dir(INOTIFY_DIR)
|
||||
|
||||
def test_uv_file_watcher(self):
|
||||
raise SkipTest('Uv watcher tests are not stable')
|
||||
try:
|
||||
w = create_file_watcher(pl=get_fallback_logger(), watcher_type='uv')
|
||||
except UvNotFound:
|
||||
raise SkipTest('Pyuv is not available')
|
||||
return self.do_test_file_watcher(w)
|
||||
|
||||
def test_tree_watcher(self):
|
||||
tw = create_tree_watcher(get_fallback_logger())
|
||||
subdir = os.path.join(INOTIFY_DIR, 'subdir')
|
||||
os.mkdir(subdir)
|
||||
if tw.watch(INOTIFY_DIR).is_dummy:
|
||||
raise SkipTest('No tree watcher available')
|
||||
self.assertTrue(tw(INOTIFY_DIR))
|
||||
self.assertFalse(tw(INOTIFY_DIR))
|
||||
changed = partial(self.do_test_for_change, tw, INOTIFY_DIR)
|
||||
open(os.path.join(INOTIFY_DIR, 'tree1'), 'w').close()
|
||||
changed()
|
||||
open(os.path.join(subdir, 'tree1'), 'w').close()
|
||||
changed()
|
||||
os.unlink(os.path.join(subdir, 'tree1'))
|
||||
changed()
|
||||
os.rmdir(subdir)
|
||||
changed()
|
||||
os.mkdir(subdir)
|
||||
changed()
|
||||
os.rename(subdir, subdir + '1')
|
||||
changed()
|
||||
shutil.rmtree(subdir + '1')
|
||||
changed()
|
||||
os.mkdir(subdir)
|
||||
f = os.path.join(subdir, 'f')
|
||||
open(f, 'w').close()
|
||||
changed()
|
||||
with open(f, 'a') as s:
|
||||
s.write(' ')
|
||||
changed()
|
||||
os.rename(f, f + '1')
|
||||
changed()
|
||||
return self.do_test_tree_watcher(tw)
|
||||
|
||||
def do_test_tree_watcher(self, tw):
|
||||
try:
|
||||
subdir = os.path.join(INOTIFY_DIR, 'subdir')
|
||||
os.mkdir(subdir)
|
||||
try:
|
||||
if tw.watch(INOTIFY_DIR).is_dummy:
|
||||
raise SkipTest('No tree watcher available')
|
||||
except UvNotFound:
|
||||
raise SkipTest('Pyuv is not available')
|
||||
self.assertTrue(tw(INOTIFY_DIR))
|
||||
self.assertFalse(tw(INOTIFY_DIR))
|
||||
changed = partial(self.do_test_for_change, tw, INOTIFY_DIR)
|
||||
open(os.path.join(INOTIFY_DIR, 'tree1'), 'w').close()
|
||||
changed()
|
||||
open(os.path.join(subdir, 'tree1'), 'w').close()
|
||||
changed()
|
||||
os.unlink(os.path.join(subdir, 'tree1'))
|
||||
changed()
|
||||
os.rmdir(subdir)
|
||||
changed()
|
||||
os.mkdir(subdir)
|
||||
changed()
|
||||
os.rename(subdir, subdir + '1')
|
||||
changed()
|
||||
shutil.rmtree(subdir + '1')
|
||||
changed()
|
||||
os.mkdir(subdir)
|
||||
f = os.path.join(subdir, 'f')
|
||||
open(f, 'w').close()
|
||||
changed()
|
||||
with open(f, 'a') as s:
|
||||
s.write(' ')
|
||||
changed()
|
||||
os.rename(f, f + '1')
|
||||
changed()
|
||||
finally:
|
||||
clear_dir(INOTIFY_DIR)
|
||||
|
||||
def test_uv_tree_watcher(self):
|
||||
raise SkipTest('Uv watcher tests are not stable')
|
||||
tw = create_tree_watcher(get_fallback_logger(), 'uv')
|
||||
return self.do_test_tree_watcher(tw)
|
||||
|
||||
|
||||
old_cwd = None
|
||||
|
|
@ -115,13 +153,7 @@ def setUpModule():
|
|||
|
||||
|
||||
def tearDownModule():
|
||||
for d in [INOTIFY_DIR]:
|
||||
for root, dirs, files in list(os.walk(d, topdown=False)):
|
||||
for file in files:
|
||||
os.remove(os.path.join(root, file))
|
||||
for dir in dirs:
|
||||
os.rmdir(os.path.join(root, dir))
|
||||
os.rmdir(d)
|
||||
shutil.rmtree(INOTIFY_DIR)
|
||||
os.chdir(old_cwd)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue